You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pu...@apache.org on 2016/11/14 05:39:46 UTC
[2/2] lens git commit: LENS-1359: Add driver hooks for user based
filtering of queries
LENS-1359: Add driver hooks for user based filtering of queries
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/02cbd6bc
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/02cbd6bc
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/02cbd6bc
Branch: refs/heads/master
Commit: 02cbd6bc3c5663a4bc97e14efd3ab64107f3d9c0
Parents: 078555c
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Nov 14 11:07:56 2016 +0530
Committer: Puneet <pu...@inmobi.com>
Committed: Mon Nov 14 11:07:56 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/lens/api/parse/Parser.java | 24 ++++
.../src/main/resources/hivedriver-default.xml | 9 +-
.../apache/lens/driver/hive/TestHiveDriver.java | 3 +-
.../lens/driver/hive/TestRemoteHiveDriver.java | 4 +-
.../lens/server/api/LensConfConstants.java | 9 +-
.../server/api/driver/AbstractLensDriver.java | 12 +-
.../server/api/driver/DriverConfiguration.java | 28 ++--
.../lens/server/api/driver/DriverQueryHook.java | 101 -------------
.../lens/server/api/driver/LensDriver.java | 1 +
.../server/api/driver/NoOpDriverQueryHook.java | 79 -----------
.../driver/hooks/ChainedDriverQueryHook.java | 130 +++++++++++++++++
.../api/driver/hooks/DriverQueryHook.java | 102 +++++++++++++
.../api/driver/hooks/NoOpDriverQueryHook.java | 80 +++++++++++
.../driver/hooks/QueryCostBasedQueryHook.java | 142 +++++++++++++++++++
.../api/driver/hooks/UserBasedQueryHook.java | 71 ++++++++++
.../query/cost/FactPartitionBasedQueryCost.java | 8 ++
.../lens/server/api/query/cost/QueryCost.java | 1 +
.../api/driver/DriverConfigurationTest.java | 58 ++++++++
.../hooks/ChainedDriverQueryHookTest.java | 82 +++++++++++
.../hooks/QueryCostBasedQueryHookTest.java | 108 ++++++++++++++
.../driver/hooks/UserBasedQueryHookTest.java | 87 ++++++++++++
.../server/api/user/MockDriverQueryHook.java | 4 +-
.../src/main/resources/lensserver-default.xml | 5 +
.../drivers/hive/hive1/hivedriver-site.xml | 2 +-
.../drivers/hive/hive2/hivedriver-site.xml | 2 +-
pom.xml | 2 +-
src/site/apt/admin/config.apt | 126 ++++++++--------
src/site/apt/admin/hivedriver-config.apt | 2 +-
28 files changed, 1006 insertions(+), 276 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java b/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java
new file mode 100644
index 0000000..afa9476
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/parse/Parser.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.api.parse;
+
+
+public interface Parser<T> {
+ T parse(String value);
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-driver-hive/src/main/resources/hivedriver-default.xml
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/resources/hivedriver-default.xml b/lens-driver-hive/src/main/resources/hivedriver-default.xml
index f5fd3bb..a13d3b0 100644
--- a/lens-driver-hive/src/main/resources/hivedriver-default.xml
+++ b/lens-driver-hive/src/main/resources/hivedriver-default.xml
@@ -35,10 +35,11 @@
</property>
<property>
- <name>lens.driver.hive.query.hook.class</name>
- <value>org.apache.lens.server.api.driver.NoOpDriverQueryHook</value>
- <description>The query hook class for hive driver. By default hook is No op. To add a hook, you should look
- at the default implementation and from there it'll be easy to derive what value can be added through a new hook
+ <name>lens.driver.hive.query.hook.classes</name>
+ <value></value>
+ <description>The query hook classes for hive driver. By default there are no hooks. To add a hook, you should look
+ at the default implementation and from there it'll be easy to derive what value can be added through a new hook.
+ Multiple hooks can be provided by providing comma seperated name of classes.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index 43b33f3..1261409 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -32,6 +32,7 @@ import org.apache.lens.cube.metadata.UpdatePeriod;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
+import org.apache.lens.server.api.driver.hooks.DriverQueryHook;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.ExplainQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
@@ -118,7 +119,7 @@ public class TestHiveDriver {
protected void createDriver() throws LensException {
driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driverConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
- driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
+ driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASSES_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
driverConf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 961ec4e..cc9fff0 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -29,10 +29,10 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.DriverQueryHook;
import org.apache.lens.server.api.driver.DriverQueryPlan;
import org.apache.lens.server.api.driver.DriverQueryStatus.DriverQueryState;
import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.driver.hooks.DriverQueryHook;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.user.MockDriverQueryHook;
@@ -148,7 +148,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driver = new HiveDriver();
driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
- driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
+ driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASSES_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
driver.configure(driverConf, "hive", "hive1");
drivers = Lists.<LensDriver>newArrayList(driver);
System.out.println("TestRemoteHiveDriver created");
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 7ccb170..7fd487c 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -20,8 +20,11 @@ package org.apache.lens.server.api;
import javax.ws.rs.core.MediaType;
+import org.apache.lens.api.parse.Parser;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.metastore.*;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
/**
* The Class LensConfConstants.
@@ -967,7 +970,7 @@ public final class LensConfConstants {
/**
* Driver hook property
*/
- public static final String DRIVER_HOOK_CLASS_SFX = "query.hook.class";
+ public static final String DRIVER_HOOK_CLASSES_SFX = "query.hook.classes";
/**
* Default driver weight
@@ -1220,6 +1223,10 @@ public final class LensConfConstants {
*/
public static final int DEFAULT_MAX_SCHEDULED_JOB_PER_USER = -1;
+ public static final String QUERY_COST_PARSER = SERVER_PFX + "query.cost.parser.class";
+ public static final Class<? extends Parser<? extends QueryCost>> DEFAULT_QUERY_COST_PARSER
+ = FactPartitionBasedQueryCost.Parser.class;
+
/**
* Maximum number of scheduled job per user.
*/
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
index 8f30aa0..91972a9 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
@@ -23,6 +23,8 @@ import static org.apache.lens.server.api.util.LensUtil.getImplementations;
import org.apache.lens.api.Priority;
import org.apache.lens.server.api.LensConfConstants;
+import org.apache.lens.server.api.driver.hooks.ChainedDriverQueryHook;
+import org.apache.lens.server.api.driver.hooks.DriverQueryHook;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.QueryContext;
@@ -85,14 +87,8 @@ public abstract class AbstractLensDriver implements LensDriver {
}
protected void loadQueryHook() throws LensException {
- try {
- queryHook = getConf().getClass(
- DRIVER_HOOK_CLASS_SFX, NoOpDriverQueryHook.class, DriverQueryHook.class
- ).newInstance();
- queryHook.setDriver(this);
- } catch (InstantiationException | IllegalAccessException e) {
- throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
- }
+ this.queryHook = ChainedDriverQueryHook.from(getConf(), DRIVER_HOOK_CLASSES_SFX);
+ queryHook.setDriver(this);
}
protected void loadRetryPolicyDecider() throws LensException {
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
index 69a1a0b..f072c73 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
@@ -34,27 +34,31 @@ public class DriverConfiguration extends Configuration {
this.driverClass = driverClass;
this.driverClassType = driverClass.getSimpleName().toLowerCase().replaceAll("driver$", "");
}
+ public DriverConfiguration(String driverType, Class<? extends AbstractLensDriver> driverClass) {
+ this.driverType = driverType;
+ this.driverClass = driverClass;
+ this.driverClassType = driverClass.getSimpleName().toLowerCase().replaceAll("driver$", "");
+ }
@Override
- public String[] getStrings(String name) {
- for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name,
- DRIVER_PFX + name, name, }) {
- String[] s = super.getStrings(key);
- if (s != null) {
- return s;
+ public String get(String name) {
+ String[] prefixes = new String[]{DRIVER_PFX + driverType + ".", DRIVER_PFX + driverClassType + ".", DRIVER_PFX, };
+ for (String prefix : prefixes) {
+ if (name.startsWith(prefix)) {
+ return getInternal(name.substring(prefix.length()));
}
}
- return null;
+ return getInternal(name);
}
- @Override
- public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) {
+ public String getInternal(String name) {
for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name,
DRIVER_PFX + name, name, }) {
- if (getTrimmed(key) != null) {
- return super.getClass(key, defaultValue, xface);
+ String val = super.get(key);
+ if (val != null) {
+ return val;
}
}
- return defaultValue;
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
deleted file mode 100644
index f8a9ee0..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryHook.java
+++ /dev/null
@@ -1,101 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- *
- */
-package org.apache.lens.server.api.driver;
-
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
-
-/**
- * Drivers should initialize a DriverQueryHook object in their initialization and expose it
- * via {@link LensDriver#getQueryHook}. Lens Server will invoke the driver hook at relevant points during
- * query execution. By default each driver exposes a {@link NoOpDriverQueryHook} which does nothing when invoked.
- *
- * The only use case I see right now is to provide a hook just after driver has been selected for a query and
- * before query is launched on the driver. One example usage for hive driver would be to add dynamic configuration or
- * stall execution of a query by looking at the final translated query itself (based on table involved, filters
- * involved, etc in the query).
- *
- * This interface is expected to evolve for some time as more needs for hook are discovered
- *
- * Note: Note if the hook updates any configuration, same should be reflected in QueryContext
- * via {@link AbstractQueryContext#updateConf(Map)} to ensure the modified configuration is persisted and is available
- * on server restarts and other bookkeeping needs.
- */
-public interface DriverQueryHook {
-
- /**
- * This setter method is called by the driver once hook instance is created. This driver information can be used while
- * extracting driver specific information form the QueryContext.
- * @param driver
- */
- void setDriver(LensDriver driver);
-
- /**
- * Called just before rewrite operation is tried on this driver
- *
- * @param ctx
- * @throws LensException
- */
- void preRewrite(AbstractQueryContext ctx) throws LensException;
-
- /**
- * Called just after a successful rewrite operation is tried on this driver
- *
- * @param ctx
- * @throws LensException
- */
- void postRewrite(AbstractQueryContext ctx) throws LensException;
-
- /**
- * Called just before estimate operation is tried on this driver
- * Note : Estimate operation will be skipped if rewrite operation fails for this driver
- *
- * @param ctx
- * @throws LensException
- */
- void preEstimate(AbstractQueryContext ctx) throws LensException;
-
- /**
- * Called just after a successful estimate operation is tried on this driver
- *
- * @param ctx
- * @throws LensException
- */
- void postEstimate(AbstractQueryContext ctx) throws LensException;
-
- /**
- * Called just after driver has been selected to execute a query.
- *
- * @param ctx
- * @throws LensException
- */
- void postDriverSelection(AbstractQueryContext ctx) throws LensException;
-
- /**
- * Called just before launching the query on the selected driver.
- * @param ctx
- * @throws LensException
- */
- void preLaunch(QueryContext ctx) throws LensException;
-
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index 1462239..0080b97 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -23,6 +23,7 @@ import java.io.Externalizable;
import org.apache.lens.api.Priority;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryPrepareHandle;
+import org.apache.lens.server.api.driver.hooks.DriverQueryHook;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
deleted file mode 100644
index 4f1f2eb..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/NoOpDriverQueryHook.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- *
- */
-package org.apache.lens.server.api.driver;
-
-import org.apache.lens.server.api.error.LensException;
-import org.apache.lens.server.api.query.AbstractQueryContext;
-import org.apache.lens.server.api.query.QueryContext;
-
-import lombok.Getter;
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class NoOpDriverQueryHook implements DriverQueryHook {
-
- @Getter
- private LensDriver driver;
-
- @Override
- public void setDriver(LensDriver driver) {
- this.driver = driver;
- log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName());
- }
-
- @Override
- public void preRewrite(AbstractQueryContext ctx) throws LensException {
- log.debug("Pre rewrite for user {}, user query: {}, driver: {}", ctx.getSubmittedUser(), ctx.getUserQuery(),
- driver.getFullyQualifiedName());
- }
-
- @Override
- public void postRewrite(AbstractQueryContext ctx) throws LensException {
- log.debug("Post rewrite for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(),
- ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver));
- }
-
- @Override
- public void preEstimate(AbstractQueryContext ctx) throws LensException {
- log.debug("Pre estimate for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(),
- ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver));
- }
-
- @Override
- public void postEstimate(AbstractQueryContext ctx) throws LensException {
- log.debug("Post estimate for user {}, user query: {}, driver: {}, driver query :{}, query cost :{}",
- ctx.getSubmittedUser(), ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver),
- ctx.getDriverQueryCost(driver));
- }
-
- @Override
- public void postDriverSelection(AbstractQueryContext ctx) throws LensException {
- log.debug("Post driver selection for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
- ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery());
- }
-
- @Override
- public void preLaunch(QueryContext ctx) {
- log.debug("Pre launch for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
- ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery());
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java
new file mode 100644
index 0000000..564028c
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHook.java
@@ -0,0 +1,130 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import java.util.List;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Lists;
+import lombok.Data;
+
+/**
+ * This hook allows chaining of different hooks. A driver can specify multiple hooks for itself by providing
+ * `query.hook.classes` in its configuration. The value should be a comma separated list of class names, each
+ * of which should be an implementation of org.apache.lens.server.api.driver.hooks.DriverQueryHook. This hook
+ * stores instances of those classes.
+ * All the methods of this class invokes the same method on each of the hooks in the chain.
+ */
+@Data
+public class ChainedDriverQueryHook extends NoOpDriverQueryHook {
+
+ private final Iterable<DriverQueryHook> hooks;
+
+ @Override
+ public void setDriver(LensDriver driver) {
+ super.setDriver(driver);
+ for (DriverQueryHook hook : hooks) {
+ hook.setDriver(driver);
+ }
+ }
+
+ @Override
+ public void preRewrite(AbstractQueryContext ctx) throws LensException {
+ super.preRewrite(ctx);
+ for (DriverQueryHook hook : hooks) {
+ hook.preRewrite(ctx);
+ }
+ }
+
+ @Override
+ public void postRewrite(AbstractQueryContext ctx) throws LensException {
+ super.postRewrite(ctx);
+ for (DriverQueryHook hook : hooks) {
+ hook.postRewrite(ctx);
+ }
+ }
+
+ @Override
+ public void preEstimate(AbstractQueryContext ctx) throws LensException {
+ super.preEstimate(ctx);
+ for (DriverQueryHook hook : hooks) {
+ hook.preEstimate(ctx);
+ }
+ }
+
+ @Override
+ public void postEstimate(AbstractQueryContext ctx) throws LensException {
+ super.postEstimate(ctx);
+ for (DriverQueryHook hook : hooks) {
+ hook.postEstimate(ctx);
+ }
+ }
+
+ @Override
+ public void postDriverSelection(AbstractQueryContext ctx) throws LensException {
+ super.postDriverSelection(ctx);
+ for (DriverQueryHook hook : hooks) {
+ hook.postDriverSelection(ctx);
+ }
+ }
+
+ @Override
+ public void preLaunch(QueryContext ctx) throws LensException {
+ super.preLaunch(ctx);
+ for (DriverQueryHook hook : hooks) {
+ hook.preLaunch(ctx);
+ }
+ }
+
+ public static ChainedDriverQueryHook from(Configuration conf, String key) throws LensException {
+ String[] classNames = conf.getStrings(key);
+ List<DriverQueryHook> hooks = Lists.newArrayList();
+ if (classNames != null) {
+ for (String className : classNames) {
+ Class<? extends DriverQueryHook> clazz;
+ try {
+ clazz = conf.getClassByName(className).asSubclass(DriverQueryHook.class);
+ } catch (ClassNotFoundException e) {
+ throw new LensException("Couldn't load class " + className, e);
+ }
+ DriverQueryHook instance;
+ try {
+ instance = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Couldn't create instance of class " + clazz.getName(), e);
+ }
+ if (instance instanceof Configurable) {
+ ((Configurable) instance).setConf(conf);
+ }
+ hooks.add(instance);
+ }
+ }
+ return new ChainedDriverQueryHook(hooks);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java
new file mode 100644
index 0000000..316bb7e
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/DriverQueryHook.java
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ * Drivers should initialize a DriverQueryHook object in their initialization and expose it
+ * via {@link LensDriver#getQueryHook}. Lens Server will invoke the driver hook at relevant points during
+ * query execution. By default each driver exposes a {@link NoOpDriverQueryHook} which does nothing when invoked.
+ *
+ * The only use case I see right now is to provide a hook just after driver has been selected for a query and
+ * before query is launched on the driver. One example usage for hive driver would be to add dynamic configuration or
+ * stall execution of a query by looking at the final translated query itself (based on table involved, filters
+ * involved, etc in the query).
+ *
+ * This interface is expected to evolve for some time as more needs for hook are discovered
+ *
+ * Note: Note if the hook updates any configuration, same should be reflected in QueryContext
+ * via {@link AbstractQueryContext#updateConf(java.util.Map)} to ensure the modified configuration is persisted and
+ * is available on server restarts and other bookkeeping needs.
+ */
+public interface DriverQueryHook {
+
+ /**
+ * This setter method is called by the driver once hook instance is created. This driver information can be used while
+ * extracting driver specific information form the QueryContext.
+ * @param driver
+ */
+ void setDriver(LensDriver driver);
+
+ /**
+ * Called just before rewrite operation is tried on this driver
+ *
+ * @param ctx
+ * @throws LensException
+ */
+ void preRewrite(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just after a successful rewrite operation is tried on this driver
+ *
+ * @param ctx
+ * @throws LensException
+ */
+ void postRewrite(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just before estimate operation is tried on this driver
+ * Note : Estimate operation will be skipped if rewrite operation fails for this driver
+ *
+ * @param ctx
+ * @throws LensException
+ */
+ void preEstimate(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just after a successful estimate operation is tried on this driver
+ *
+ * @param ctx
+ * @throws LensException
+ */
+ void postEstimate(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just after driver has been selected to execute a query.
+ *
+ * @param ctx
+ * @throws LensException
+ */
+ void postDriverSelection(AbstractQueryContext ctx) throws LensException;
+
+ /**
+ * Called just before launching the query on the selected driver.
+ * @param ctx
+ * @throws LensException
+ */
+ void preLaunch(QueryContext ctx) throws LensException;
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java
new file mode 100644
index 0000000..a6af091
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/NoOpDriverQueryHook.java
@@ -0,0 +1,80 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
+
+import lombok.Getter;
+import lombok.extern.slf4j.Slf4j;
+
+@Slf4j
+public class NoOpDriverQueryHook implements DriverQueryHook {
+
+ @Getter
+ private LensDriver driver;
+
+ @Override
+ public void setDriver(LensDriver driver) {
+ this.driver = driver;
+ log.debug("The Driver for this driver query hook is {}", driver.getFullyQualifiedName());
+ }
+
+ @Override
+ public void preRewrite(AbstractQueryContext ctx) throws LensException {
+ log.debug("Pre rewrite for user {}, user query: {}, driver: {}", ctx.getSubmittedUser(), ctx.getUserQuery(),
+ driver.getFullyQualifiedName());
+ }
+
+ @Override
+ public void postRewrite(AbstractQueryContext ctx) throws LensException {
+ log.debug("Post rewrite for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(),
+ ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver));
+ }
+
+ @Override
+ public void preEstimate(AbstractQueryContext ctx) throws LensException {
+ log.debug("Pre estimate for user {}, user query: {}, driver: {}, driver query :{}", ctx.getSubmittedUser(),
+ ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver));
+ }
+
+ @Override
+ public void postEstimate(AbstractQueryContext ctx) throws LensException {
+ log.debug("Post estimate for user {}, user query: {}, driver: {}, driver query :{}, query cost :{}",
+ ctx.getSubmittedUser(), ctx.getUserQuery(), driver.getFullyQualifiedName(), ctx.getDriverQuery(driver),
+ ctx.getDriverQueryCost(driver));
+ }
+
+ @Override
+ public void postDriverSelection(AbstractQueryContext ctx) throws LensException {
+ log.debug("Post driver selection for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
+ ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery());
+ }
+
+ @Override
+ public void preLaunch(QueryContext ctx) throws LensException {
+ log.debug("Pre launch for user {}, user query: {}, driver {}, driver query: {}", ctx.getSubmittedUser(),
+ ctx.getUserQuery(), ctx.getSelectedDriver().getFullyQualifiedName(), ctx.getSelectedDriverQuery());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java
new file mode 100644
index 0000000..8cfe472
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHook.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import static org.apache.lens.server.api.LensConfConstants.DEFAULT_QUERY_COST_PARSER;
+import static org.apache.lens.server.api.LensConfConstants.QUERY_COST_PARSER;
+
+import org.apache.lens.api.parse.Parser;
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import com.google.common.collect.*;
+
+/**
+ * This hook allows for a driver to discard queries based on the cost of the query. This works on two
+ * configurations values, among which either, both or none can be set. The configuration values are picked from
+ * driver configuration. The driver config can set `allowed.range.sets` as a mathematical range of costs that
+ * are allowed on this driver. Queries having cost among these will be allowed
+ * and the rest will be discarded. On the other hand, the driver can set `disallowed.range.sets` as a
+ * mathematical range of costs that are disallowed on this driver. Queries having range among these
+ * will be rejected, and the rest will be allowed.
+ *
+ * Mathematical range looks like the following:
+ *
+ * (10, 20] U (30, 40) U [1000, )
+ *
+ *
+ */
+public class QueryCostBasedQueryHook<T extends QueryCost<T>> extends NoOpDriverQueryHook {
+ public static final String ALLOWED_RANGE_SETS = "allowed.range.sets";
+ public static final String DISALLOWED_RANGE_SETS = "disallowed.range.sets";
+ private RangeSet<T> allowedCosts;
+ private Parser<T> parser;
+
+ public RangeSet<T> parseAllowedRangeSet(String rangeStr) {
+ RangeSet<T> parsed = parseRangeSet(rangeStr);
+ if (parsed == null) {
+ TreeRangeSet<T> set = TreeRangeSet.create();
+ set.add(Range.<T>all());
+ return set;
+ } else {
+ return parsed;
+ }
+ }
+
+ public RangeSet<T> parseDisallowedRangeSet(String rangeStr) {
+ RangeSet<T> parsed = parseRangeSet(rangeStr);
+ if (parsed == null) {
+ return TreeRangeSet.create();
+ } else {
+ return parsed;
+ }
+ }
+
+ public RangeSet<T> parseRangeSet(String rangeStr) {
+ if (rangeStr == null) {
+ return null;
+ }
+ RangeSet<T> set = TreeRangeSet.create();
+ for (String range : rangeStr.split("U")) {
+ set.add(parseRange(range));
+ }
+ return set;
+ }
+
+ public Range<T> parseRange(String rangeStr) {
+ if (rangeStr == null) {
+ return null;
+ }
+ rangeStr = rangeStr.trim();
+ BoundType lowerBound, upperBound;
+ if (rangeStr.startsWith("[")) {
+ lowerBound = BoundType.CLOSED;
+ } else if (rangeStr.startsWith("(")) {
+ lowerBound = BoundType.OPEN;
+ } else {
+ throw new IllegalArgumentException("Range should start with either ( or [");
+ }
+ if (rangeStr.endsWith("]")) {
+ upperBound = BoundType.CLOSED;
+ } else if (rangeStr.endsWith(")")) {
+ upperBound = BoundType.OPEN;
+ } else {
+ throw new IllegalArgumentException("Range should end with either ) or ]");
+ }
+ String[] pair = rangeStr.substring(1, rangeStr.length() - 1).split(",");
+ String leftStr = pair[0].trim();
+ String rightStr = pair[1].trim();
+ if (leftStr.isEmpty() && rightStr.isEmpty()) {
+ return Range.all();
+ } else if (leftStr.isEmpty()) {
+ return Range.upTo(parser.parse(rightStr), upperBound);
+ } else if (rightStr.isEmpty()) {
+ return Range.downTo(parser.parse(leftStr), lowerBound);
+ } else {
+ return Range.range(parser.parse(leftStr), lowerBound, parser.parse(rightStr), upperBound);
+ }
+ }
+
+ @Override
+ public void setDriver(LensDriver driver) {
+ super.setDriver(driver);
+ Class<? extends Parser<T>> parserClass = (Class<? extends Parser<T>>) driver.getConf().getClass(QUERY_COST_PARSER,
+ DEFAULT_QUERY_COST_PARSER, Parser.class);
+ try {
+ this.parser = parserClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new IllegalArgumentException("Couldn't initialize query cost parser from class " + parserClass);
+ }
+ allowedCosts = parseAllowedRangeSet(driver.getConf().get(ALLOWED_RANGE_SETS));
+ allowedCosts.removeAll(parseDisallowedRangeSet(driver.getConf().get(DISALLOWED_RANGE_SETS)));
+ }
+
+ @Override
+ public void postEstimate(AbstractQueryContext ctx) throws LensException {
+ if (!allowedCosts.contains((T) ctx.getDriverQueryCost(getDriver()))) {
+ throw new LensException("Driver " + getDriver() + " doesn't accept query "
+ + "with cost " + ctx.getSelectedDriverQueryCost() + ". Allowed ranges: " + allowedCosts);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java
new file mode 100644
index 0000000..1f51db0
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHook.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import java.util.Set;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+import com.google.common.collect.Sets;
+
+/**
+ * This hook allows for a driver to discard queries based on the submitter of the query. This works on two
+ * configurations values, among which either or none can be set. The configuration values are picked from
+ * driver configuration. The driver config can set `allowed.users` as a comma separated list of users that
+ * are allowed to submit queries on this driver. Queries from only those users will be allowed
+ * and the rest will be discarded. On the other hand, the driver can set `disallowed.users` as a
+ * comma separated list of users that are disallowed to submit queries on this driver. Queries from these users
+ * will be rejected, while queries from any other user will be accepted.
+ */
+public class UserBasedQueryHook extends NoOpDriverQueryHook {
+ public static final String ALLOWED_USERS = "allowed.users";
+ public static final String DISALLOWED_USERS = "disallowed.users";
+ private Set<String> allowedUsers;
+ private Set<String> disallowedUsers;
+
+ @Override
+ public void setDriver(LensDriver driver) {
+ super.setDriver(driver);
+ String[] allowedUsers = driver.getConf().getStrings(ALLOWED_USERS);
+ String[] disallowedUsers = driver.getConf().getStrings(DISALLOWED_USERS);
+ if (allowedUsers != null && disallowedUsers != null) {
+ throw new IllegalStateException("You can't specify both allowed users and disallowed users");
+ }
+ this.allowedUsers = allowedUsers == null ? null : Sets.newHashSet(allowedUsers);
+ this.disallowedUsers = disallowedUsers == null ? null : Sets.newHashSet(disallowedUsers);
+ }
+
+ private String getErrorMessage(AbstractQueryContext ctx) {
+ return "User " + ctx.getSubmittedUser() + " not allowed to submit query on driver "
+ + getDriver().getFullyQualifiedName();
+ }
+
+ @Override
+ public void preRewrite(AbstractQueryContext ctx) throws LensException {
+ if ((allowedUsers != null && !allowedUsers.contains(ctx.getSubmittedUser()))
+ || (disallowedUsers != null && disallowedUsers.contains(ctx.getSubmittedUser()))) {
+ throw new LensException(getErrorMessage(ctx));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
index 4768550..eba8f0d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/FactPartitionBasedQueryCost.java
@@ -69,4 +69,12 @@ public class FactPartitionBasedQueryCost implements QueryCost<FactPartitionBased
public String toString() {
return getQueryCostType() + "(" + getEstimatedResourceUsage() + ")";
}
+
+ public static class Parser implements org.apache.lens.api.parse.Parser<FactPartitionBasedQueryCost> {
+
+ @Override
+ public FactPartitionBasedQueryCost parse(String value) {
+ return new FactPartitionBasedQueryCost(Double.parseDouble(value));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
index 4712f11..9d7320a 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/cost/QueryCost.java
@@ -37,4 +37,5 @@ public interface QueryCost<T extends QueryCost> extends Comparable<T> {
long getEstimatedExecTimeMillis() throws UnsupportedOperationException;
double getEstimatedResourceUsage() throws UnsupportedOperationException;
+
}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java
new file mode 100644
index 0000000..2a9cac1
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/DriverConfigurationTest.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+import static org.apache.lens.server.api.LensConfConstants.DRIVER_PFX;
+
+import static org.testng.Assert.*;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class DriverConfigurationTest {
+ public static final String DRIVER_TYPE = "type";
+ public static final String DRIVER_CLASS_TYPE = "mock";
+ public static final String KEY = "key";
+ public static final String VALUE = "value";
+ public static final String[] PREFIXES = new String[]{
+ DRIVER_PFX + DRIVER_TYPE + ".",
+ DRIVER_PFX + DRIVER_CLASS_TYPE + ".",
+ DRIVER_PFX,
+ "",
+ };
+
+ @DataProvider
+ public Object[][] keyData() {
+ Object[][] data = new Object[16][2];
+ for (int i = 0; i < 4; i++) {
+ for (int j = 0; j < 4; j++) {
+ data[i * 4 + j][0] = PREFIXES[i] + KEY;
+ data[i * 4 + j][1] = PREFIXES[j] + KEY;
+ }
+ }
+ return data;
+ }
+
+ @Test(dataProvider = "keyData")
+ public void testSetAndGet(String keyToSet, String keyToGet) {
+ DriverConfiguration conf = new DriverConfiguration(DRIVER_TYPE, MockDriver.class);
+ conf.set(keyToSet, VALUE);
+ assertEquals(conf.get(keyToGet), VALUE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java
new file mode 100644
index 0000000..3ea2c42
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/ChainedDriverQueryHookTest.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.*;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+
+public class ChainedDriverQueryHookTest {
+ private AbstractQueryContext ctx;
+ ChainedDriverQueryHook hook;
+ private LensDriver driver;
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ driver = mock(LensDriver.class);
+ Configuration conf = new Configuration();
+ when(driver.getConf()).thenReturn(conf);
+ ctx = mock(AbstractQueryContext.class);
+ conf.set(UserBasedQueryHook.ALLOWED_USERS, "user1,user2");
+ conf.set(QueryCostBasedQueryHook.ALLOWED_RANGE_SETS, "[0, 10)");
+ conf.set("chain", UserBasedQueryHook.class.getName() + "," + QueryCostBasedQueryHook.class.getName());
+ hook = ChainedDriverQueryHook.from(conf, "chain");
+ hook.setDriver(driver);
+ }
+
+ @DataProvider
+ public Object[][] provideData() {
+ return new Object[][]{
+ {"user1", 0.0, true}, // allowed by both
+ {"user3", 0.0, false}, // disallowed by former
+ {"user1", 10.0, false}, // disallowed by latter
+ {"user3", 11.0, false}, // disallowed by both
+ };
+ }
+
+ @Test(dataProvider = "provideData")
+ public void testChaining(String user, Double cost, boolean allowed) {
+ when(ctx.getDriverQueryCost(driver)).thenReturn(new FactPartitionBasedQueryCost(cost));
+ when(ctx.getSubmittedUser()).thenReturn(user);
+ try {
+ hook.preRewrite(ctx);
+ hook.postRewrite(ctx);
+ hook.preEstimate(ctx);
+ hook.postEstimate(ctx);
+ assertTrue(allowed);
+ } catch (LensException e) {
+ assertFalse(allowed);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java
new file mode 100644
index 0000000..80cddf3
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/QueryCostBasedQueryHookTest.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.RangeSet;
+
+
+public class QueryCostBasedQueryHookTest {
+
+ private QueryCostBasedQueryHook hook = new QueryCostBasedQueryHook();
+ private AbstractQueryContext ctx;
+ private LensDriver driver;
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ driver = mock(LensDriver.class);
+ Configuration conf = new Configuration();
+ conf.set(QueryCostBasedQueryHook.ALLOWED_RANGE_SETS, "[10, 100]");
+ conf.set(QueryCostBasedQueryHook.DISALLOWED_RANGE_SETS, "[40, 50]");
+ when(driver.getConf()).thenReturn(conf);
+ ctx = mock(AbstractQueryContext.class);
+ hook.setDriver(driver);
+ }
+
+ @DataProvider
+ public Object[][] provideParsingData() {
+ return new Object[][]{
+ {"(, 5) U [5, 10) U [100, )", new Double[]{0.0, 1.0, 5.0, 100.0}, new Double[]{10.0, 99.0}},
+ {"(, )", new Double[]{0.0, 1.0, 10.0, 100.0, 1000000.0}, new Double[]{}},
+ {"(5, 10)", new Double[]{6.0, 7.0, 9.9}, new Double[]{2.0, 5.0, 10.0, 20.0}},
+ {"[5, 10]", new Double[]{5.0, 6.0, 7.0, 9.9, 10.0}, new Double[]{2.0, 4.9, 10.1, 20.0}},
+ };
+ }
+
+ @Test(dataProvider = "provideParsingData")
+ public void testParse(String rangeString, Double[] includes, Double[] excludes) {
+ RangeSet<FactPartitionBasedQueryCost> range = hook.parseRangeSet(rangeString);
+ for (Double cost : includes) {
+ assertTrue(range.contains(new FactPartitionBasedQueryCost(cost)));
+ }
+ for (Double cost : excludes) {
+ assertFalse(range.contains(new FactPartitionBasedQueryCost(cost)));
+ }
+ }
+
+ @DataProvider
+ public Object[][] provideData() {
+ return new Object[][]{
+ {5.0, false},
+ {10.0, true},
+ {30.0, true},
+ {40.0, false},
+ {45.0, false},
+ {50.0, false},
+ {51.0, true},
+ {99.0, true},
+ {100.0, true},
+ {101.0, false},
+ {1001.0, false},
+ };
+ }
+
+ @Test(dataProvider = "provideData")
+ public void testHook(Double cost, boolean success)
+ throws Exception {
+ when(ctx.getDriverQueryCost(driver)).thenReturn(new FactPartitionBasedQueryCost(cost));
+ try {
+ hook.postEstimate(ctx);
+ assertTrue(success);
+ } catch (Exception e) {
+ assertFalse(success);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java
new file mode 100644
index 0000000..47060f8
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/hooks/UserBasedQueryHookTest.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ *
+ */
+package org.apache.lens.server.api.driver.hooks;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.testng.Assert.*;
+
+import org.apache.lens.server.api.driver.LensDriver;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+public class UserBasedQueryHookTest {
+
+ private LensDriver driver;
+ private Configuration conf;
+ private UserBasedQueryHook hook = new UserBasedQueryHook();
+ private AbstractQueryContext ctx;
+
+ @BeforeMethod
+ public void setUp() throws Exception {
+ driver = mock(LensDriver.class);
+ conf = new Configuration();
+ when(driver.getConf()).thenReturn(conf);
+ ctx = mock(AbstractQueryContext.class);
+ }
+
+ @DataProvider
+ public Object[][] provideData() {
+ return new Object[][]{
+ {"a,b", "b,c", "d", false}, // disallow setting both
+ {"a,b", null, "c", false}, // disallowed
+ {null, "a,b", "c", true}, // not disallowed
+ {"a,b", null, "a", true}, // allowed
+ {null, "a,b", "a", false}, // not allowed
+ {null, null, "a", true}, // no restrictions
+ {"", "", "a", true}, // null and blank string are same
+ };
+ }
+
+ @Test(dataProvider = "provideData")
+ public void testPreRewrite(String allowedString, String disallowedString, String user, boolean success)
+ throws Exception {
+ if (allowedString != null) {
+ conf.set(UserBasedQueryHook.ALLOWED_USERS, allowedString);
+ } else {
+ conf.unset(UserBasedQueryHook.ALLOWED_USERS);
+ }
+ if (disallowedString != null) {
+ conf.set(UserBasedQueryHook.DISALLOWED_USERS, disallowedString);
+ } else {
+ conf.unset(UserBasedQueryHook.DISALLOWED_USERS);
+ }
+ when(ctx.getSubmittedUser()).thenReturn(user);
+ try {
+ hook.setDriver(driver);
+ hook.preRewrite(ctx);
+ assertTrue(success);
+ } catch (Exception e) {
+ assertFalse(success);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
index f70979a..9dfb3b6 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/user/MockDriverQueryHook.java
@@ -20,7 +20,7 @@ package org.apache.lens.server.api.user;
import java.util.HashMap;
-import org.apache.lens.server.api.driver.NoOpDriverQueryHook;
+import org.apache.lens.server.api.driver.hooks.NoOpDriverQueryHook;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.QueryContext;
@@ -40,7 +40,7 @@ public class MockDriverQueryHook extends NoOpDriverQueryHook {
public static final String POST_ESTIMATE = "POST_ESTIMATE";
@Override
- public void preLaunch(QueryContext ctx) {
+ public void preLaunch(QueryContext ctx) throws LensException {
super.preLaunch(ctx);
ctx.getSelectedDriverConf().set(KEY_PRE_LAUNCH, VALUE_PRE_LAUNCH);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server/src/main/resources/lensserver-default.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/main/resources/lensserver-default.xml b/lens-server/src/main/resources/lensserver-default.xml
index e652a0f..261fa52 100644
--- a/lens-server/src/main/resources/lensserver-default.xml
+++ b/lens-server/src/main/resources/lensserver-default.xml
@@ -938,6 +938,11 @@
</description>
</property>
<property>
+ <name>lens.server.query.cost.parser.class</name>
+ <value>org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost$Parser</value>
+ <description>The Query cost parser class. Default query cost class used is FactPartitionBasedQueryCost</description>
+ </property>
+ <property>
<name>lens.cube.metastore.enable.datacompleteness.check</name>
<value>false</value>
<description>This property is to enable Data Completeness Checks while resolving partitions.</description>
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
index 50985b5..1f0ff43 100644
--- a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
+++ b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -76,7 +76,7 @@
</property>
<property>
- <name>lens.driver.hive.query.hook.class</name>
+ <name>lens.driver.hive.query.hook.classes</name>
<value>org.apache.lens.server.api.user.MockDriverQueryHook</value>
<description>The query hook class for hive driver.
</description>
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
index b485100..eb902e0 100644
--- a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
+++ b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
@@ -76,7 +76,7 @@
</property>
<property>
- <name>lens.driver.hive.query.hook.class</name>
+ <name>lens.driver.hive.query.hook.classes</name>
<value>org.apache.lens.server.api.user.MockDriverQueryHook</value>
<description>The query hook class for hive driver.
</description>
http://git-wip-us.apache.org/repos/asf/lens/blob/02cbd6bc/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 29c59d3..8ea64b7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,7 +58,7 @@
<commons.collections.version>3.2.1</commons.collections.version>
<joda.time.version>2.0</joda.time.version>
<quartz.version>2.2.2</quartz.version>
- <guava.version>13.0.1</guava.version>
+ <guava.version>14.0.1</guava.version>
<lombok.version>1.16.6</lombok.version>
<lombok.maven.plugin.version>1.16.4.1</lombok.maven.plugin.version>
<typesafe.config.version>1.2.1</typesafe.config.version>