You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2016/10/17 05:45:17 UTC
[4/4] lens git commit: LENS-743: Query retry framework for retrying
upon transient failures
LENS-743: Query retry framework for retrying upon transient failures
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/38ab6c60
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/38ab6c60
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/38ab6c60
Branch: refs/heads/master
Commit: 38ab6c6082b6221502daac979551e8c5fca72241
Parents: 182f6dc
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Oct 17 11:13:34 2016 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Oct 17 11:13:34 2016 +0530
----------------------------------------------------------------------
checkstyle/src/main/resources/checkstyle.xml | 2 +-
.../apache/lens/api/query/FailedAttempt.java | 68 ++++++
.../org/apache/lens/api/query/LensQuery.java | 6 +
.../org/apache/lens/api/query/QueryStatus.java | 34 ++-
.../lens/api/jaxb/YAMLToStringStrategyTest.java | 14 +-
.../org.apache.lens.api.query.LensQuery.xml | 5 -
.../org.apache.lens.api.query.QueryStatus.xml | 5 -
.../org/apache/lens/driver/es/ESDriver.java | 29 +--
lens-driver-es/src/test/resources/hive-site.xml | 5 +
.../org/apache/lens/driver/hive/HiveDriver.java | 65 +-----
.../apache/lens/driver/hive/TestHiveDriver.java | 2 +-
.../lens/driver/hive/TestRemoteHiveDriver.java | 2 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 69 ++----
.../driver/jdbc/JDBCDriverConfConstants.java | 6 -
.../jdbc/MaxJDBCConnectionCheckConstraint.java | 16 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 4 +-
.../regression/core/constants/DriverConfig.java | 4 +-
.../lens/server/api/LensConfConstants.java | 21 +-
.../server/api/common/BackOffRetryHandler.java | 71 -------
.../lens/server/api/common/FailureContext.java | 43 ----
...FibonacciExponentialBackOffRetryHandler.java | 77 -------
.../common/OperationRetryHandlerFactory.java | 41 ----
.../server/api/driver/AbstractLensDriver.java | 56 ++++-
.../server/api/driver/DriverConfiguration.java | 60 ++++++
.../server/api/driver/DriverQueryStatus.java | 48 ++---
.../lens/server/api/driver/LensDriver.java | 3 +
.../server/api/query/AbstractQueryContext.java | 2 +-
.../api/query/DriverSelectorQueryContext.java | 17 +-
.../server/api/query/FinishedLensQuery.java | 15 +-
.../lens/server/api/query/PriorityChange.java | 40 ----
.../lens/server/api/query/QueryAccepted.java | 39 ----
.../lens/server/api/query/QueryCancelled.java | 51 -----
.../lens/server/api/query/QueryClosed.java | 50 -----
.../lens/server/api/query/QueryContext.java | 68 ++++--
.../lens/server/api/query/QueryEnded.java | 87 --------
.../lens/server/api/query/QueryEvent.java | 93 --------
.../lens/server/api/query/QueryExecuted.java | 41 ----
.../lens/server/api/query/QueryFailed.java | 50 -----
.../lens/server/api/query/QueryLaunched.java | 41 ----
.../lens/server/api/query/QueryQueued.java | 59 ------
.../lens/server/api/query/QueryRejected.java | 39 ----
.../lens/server/api/query/QueryRunning.java | 41 ----
.../lens/server/api/query/QuerySuccess.java | 47 ----
.../server/api/query/QueuePositionChange.java | 39 ----
.../lens/server/api/query/StatusChange.java | 52 -----
.../api/query/StatusUpdateFailureContext.java | 45 ++++
.../query/comparators/ChainedComparator.java | 41 ++++
.../query/comparators/FIFOQueryComparator.java | 29 +++
.../comparators/MoreRetriesFirstComparator.java | 30 +++
.../api/query/comparators/QueryComparator.java | 28 +++
.../query/comparators/QueryCostComparator.java | 30 +++
.../comparators/QueryPriorityComparator.java | 29 +++
.../MaxConcurrentDriverQueriesConstraint.java | 64 ++++--
.../constraint/QueryLaunchingConstraint.java | 4 +-
.../server/api/query/events/PriorityChange.java | 40 ++++
.../server/api/query/events/QueryAccepted.java | 39 ++++
.../server/api/query/events/QueryCancelled.java | 52 +++++
.../server/api/query/events/QueryClosed.java | 51 +++++
.../server/api/query/events/QueryEnded.java | 88 ++++++++
.../server/api/query/events/QueryEvent.java | 93 ++++++++
.../server/api/query/events/QueryExecuted.java | 41 ++++
.../server/api/query/events/QueryFailed.java | 51 +++++
.../server/api/query/events/QueryLaunched.java | 41 ++++
.../server/api/query/events/QueryQueued.java | 59 ++++++
.../api/query/events/QueryQueuedForRetry.java | 41 ++++
.../server/api/query/events/QueryRejected.java | 39 ++++
.../server/api/query/events/QueryRunning.java | 41 ++++
.../server/api/query/events/QuerySuccess.java | 48 +++++
.../api/query/events/QueuePositionChange.java | 39 ++++
.../server/api/query/events/StatusChange.java | 52 +++++
.../server/api/retry/BackOffRetryHandler.java | 74 +++++++
.../api/retry/ChainedRetryPolicyDecider.java | 72 +++++++
.../api/retry/DefaultRetryPolicyDecider.java | 27 +++
.../lens/server/api/retry/FailureContext.java | 38 ++++
...FibonacciExponentialBackOffRetryHandler.java | 78 +++++++
.../server/api/retry/ImmediateRetryHandler.java | 46 ++++
.../lens/server/api/retry/NoRetryHandler.java | 35 +++
.../api/retry/OperationRetryHandlerFactory.java | 42 ++++
.../server/api/retry/RetryPolicyDecider.java | 25 +++
.../apache/lens/server/api/util/LensUtil.java | 34 +--
.../TestExponentialBackOffRetryHandler.java | 52 -----
.../lens/server/api/driver/MockDriver.java | 4 +-
.../lens/server/api/query/TestQueryContext.java | 3 +-
.../comparators/ChainedComparatorTest.java | 78 +++++++
...axConcurrentDriverQueriesConstraintTest.java | 43 +++-
.../TestExponentialBackOffRetryHandler.java | 55 +++++
lens-server/pom.xml | 2 +-
.../lens/server/metrics/MetricsServiceImpl.java | 2 +-
.../lens/server/query/FIFOQueryComparator.java | 33 ---
.../apache/lens/server/query/LensServerDAO.java | 98 +++++++--
.../lens/server/query/QueryComparator.java | 28 ---
.../lens/server/query/QueryCostComparator.java | 39 ----
.../lens/server/query/QueryEndHttpNotifier.java | 4 +-
.../lens/server/query/QueryEndNotifier.java | 2 +-
.../server/query/QueryEventHttpNotifier.java | 4 +-
.../server/query/QueryExecutionServiceImpl.java | 152 ++++++++++---
.../QueryExecutionStatisticsGenerator.java | 2 +-
.../server/query/QueryPriorityComparator.java | 38 ----
.../lens/server/query/ResultFormatter.java | 1 +
...DefaultQueryLaunchingConstraintsChecker.java | 19 +-
.../RetryPolicyToConstraingAdapter.java | 40 ++++
.../TotalQueryCostCeilingConstraint.java | 12 +-
.../apache/lens/server/rewrite/RewriteUtil.java | 2 +-
.../scheduler/SchedulerQueryEventListener.java | 2 +-
.../server/scheduler/SchedulerServiceImpl.java | 2 +-
.../query/QueryContextComparatorTest.java | 20 +-
.../lens/server/query/TestEventService.java | 2 +-
.../apache/lens/server/query/TestLensDAO.java | 23 +-
.../TestQueryIndependenceFromSessionClose.java | 5 +
.../query/TestQueryNotifictaionResource.java | 4 +-
.../lens/server/query/TestQueryService.java | 72 ++++---
.../server/query/collect/QueryCollectUtil.java | 4 +-
...ultQueryLaunchingConstraintsCheckerTest.java | 18 +-
.../ThreadSafeEstimatedQueryCollectionTest.java | 11 +-
.../TotalQueryCostCeilingConstraintTest.java | 18 +-
.../query/retry/MockDriverForRetries.java | 82 +++++++
.../query/retry/MockRetryPolicyDecider.java | 35 +++
.../lens/server/query/retry/QueryRetryTest.java | 212 +++++++++++++++++++
.../retry/TestServerRetryPolicyDecider.java | 32 +++
.../scheduler/util/SchedulerTestUtils.java | 2 +-
.../failing-query-driver-site.xml | 32 +++
.../retry/double_failure/driver-site.xml | 31 +++
.../retry/single_failure/driver-site.xml | 31 +++
.../retry/triple_failure/driver-site.xml | 31 +++
124 files changed, 2988 insertions(+), 1607 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/checkstyle/src/main/resources/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml
index 17e55ea..634234d 100644
--- a/checkstyle/src/main/resources/checkstyle.xml
+++ b/checkstyle/src/main/resources/checkstyle.xml
@@ -62,7 +62,7 @@
</module>
<module name="FileLength">
- <property name="max" value="3500"/>
+ <property name="max" value="4000"/>
</module>
<module name="FileTabCharacter"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java b/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java
new file mode 100644
index 0000000..5cf2dc3
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.api.query;
+
+import java.io.Serializable;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import lombok.*;
+
+
+@XmlRootElement
+@AllArgsConstructor
+/**
+ * Instantiates a new query status.
+ */
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@EqualsAndHashCode
+public class FailedAttempt implements Comparable<FailedAttempt>, Serializable {
+ @XmlElement
+ @Getter
+ private String driverName;
+ @XmlElement
+ @Getter
+ private double progress;
+ @XmlElement
+ @Getter
+ private String progressMessage;
+ @XmlElement
+ @Getter
+ private String errorMessage;
+ @XmlElement
+ @Getter
+ private Long driverStartTime;
+ @XmlElement
+ @Getter
+ private Long driverFinishTime;
+
+ @Override
+ public int compareTo(FailedAttempt o) {
+ if (o == null) {
+ return 1;
+ }
+ int startCompare = driverStartTime.compareTo(o.getDriverStartTime());
+ if (startCompare != 0) {
+ return startCompare;
+ }
+ return driverFinishTime.compareTo(o.getDriverFinishTime());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
index 7d079a9..d7b8963 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
@@ -21,6 +21,8 @@
*/
package org.apache.lens.api.query;
+import java.util.List;
+
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@@ -215,6 +217,10 @@ public class LensQuery extends ToYAMLString {
@Getter
private String queryName;
+ @XmlElement
+ @Getter
+ private List<FailedAttempt> failedAttempts;
+
/**
* @return error code in case of query failures
*/
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index daedbf9..3c6ecf7 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -62,11 +62,11 @@ import lombok.*;
@NoArgsConstructor(access = AccessLevel.PROTECTED)
public class QueryStatus extends ToYAMLString implements Serializable {
+
/**
* The Constant serialVersionUID.
*/
private static final long serialVersionUID = 1L;
-
/**
* The Enum Status.
*/
@@ -106,6 +106,12 @@ public class QueryStatus extends ToYAMLString implements Serializable {
EXECUTED,
/**
+ * This state is when depending on retry policy, either the query moves to QUEUED (in case retries are to be done),
+ * or to FAILED.
+ */
+ FAILING,
+
+ /**
* The successful.
* At this point all operations related to the query are finished successfully by driver and server.
*/
@@ -177,7 +183,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
@Setter
private String errorMessage;
- @XmlElement
+ @Getter
private LensErrorTO lensErrorTO;
public boolean finished() {
@@ -205,6 +211,10 @@ public class QueryStatus extends ToYAMLString implements Serializable {
return status.equals(Status.FAILED);
}
+ public boolean failing() {
+ return status.equals(Status.FAILING);
+ }
+
public boolean cancelled() {
return status.equals(Status.CANCELED);
}
@@ -232,7 +242,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
case QUEUED:
switch (newState) {
case LAUNCHED:
- case FAILED:
+ case FAILING:
case CANCELED:
return true;
}
@@ -242,7 +252,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
case LAUNCHED:
case RUNNING:
case CANCELED:
- case FAILED:
+ case FAILING:
case EXECUTED:
return true;
}
@@ -251,7 +261,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
switch (newState) {
case RUNNING:
case CANCELED:
- case FAILED:
+ case FAILING:
case EXECUTED:
return true;
}
@@ -260,17 +270,25 @@ public class QueryStatus extends ToYAMLString implements Serializable {
switch (newState) {
case EXECUTED:
case SUCCESSFUL:
- case FAILED:
+ case FAILING:
case CANCELED:
return true;
}
break;
+ case FAILING:
+ switch(newState) {
+ case QUEUED:
+ case FAILED:
+ return true;
+ }
+ break;
case FAILED:
case CANCELED:
case SUCCESSFUL:
if (Status.CLOSED.equals(newState)) {
return true;
}
+ break;
default:
// fall-through
}
@@ -294,4 +312,8 @@ public class QueryStatus extends ToYAMLString implements Serializable {
public String getLensErrorTOErrorMsg() {
return (this.lensErrorTO != null) ? this.lensErrorTO.getMessage() : null;
}
+
+ public static QueryStatus getQueuedStatus() {
+ return new QueryStatus(0.0, null, Status.QUEUED, "Query is queued", false, null, null, null);
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
----------------------------------------------------------------------
diff --git a/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java b/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
index 04d9ad6..eb568a7 100644
--- a/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
+++ b/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
@@ -147,11 +147,15 @@ public class YAMLToStringStrategyTest {
return name.endsWith("xml");
}
})) {
- Class<?> clazz = Class.forName(fn.substring(0, fn.length() - 4));
- Object unmarshalled = clazz.cast(new LensJAXBContext(clazz)
- .createUnmarshaller().unmarshal(getClass().getResourceAsStream("/toString/" + fn)));
- String toString = readYAML("/toString/" + fn.replaceAll("xml$", "yaml"));
- dataList.add(new ToStringTestData(fn, unmarshalled, toString));
+ try {
+ Class<?> clazz = Class.forName(fn.substring(0, fn.length() - 4));
+ Object unmarshalled = clazz.cast(new LensJAXBContext(clazz)
+ .createUnmarshaller().unmarshal(getClass().getResourceAsStream("/toString/" + fn)));
+ String toString = readYAML("/toString/" + fn.replaceAll("xml$", "yaml"));
+ dataList.add(new ToStringTestData(fn, unmarshalled, toString));
+ } catch (Throwable th) {
+ dataList.add(new ToStringTestData(fn, th));
+ }
}
return dataList;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
----------------------------------------------------------------------
diff --git a/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml b/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
index a681bca..fc609d9 100644
--- a/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
+++ b/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
@@ -34,11 +34,6 @@
<status>RUNNING</status>
<statusMessage>query running</statusMessage>
<isResultSetAvailable>false</isResultSetAvailable>
- <lensErrorTO>
- <code>9999</code>
- <message>error message</message>
- <stackTrace>random stack trace</stackTrace>
- </lensErrorTO>
<errorMessage>error message</errorMessage>
<progressMessage>progress message</progressMessage>
<queueNumber>11</queueNumber>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
----------------------------------------------------------------------
diff --git a/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml b/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
index 70f9c93..b1957d7 100644
--- a/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
+++ b/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
@@ -24,11 +24,6 @@
<status>RUNNING</status>
<statusMessage>query running</statusMessage>
<isResultSetAvailable>false</isResultSetAvailable>
- <lensErrorTO>
- <code>9999</code>
- <message>error message</message>
- <stackTrace>random stack trace</stackTrace>
- </lensErrorTO>
<errorMessage>error message</errorMessage>
<progressMessage>progress message</progressMessage>
<queueNumber>11</queueNumber>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
index fceabea..e735c0c 100644
--- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
@@ -42,8 +42,6 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
-import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
@@ -60,9 +58,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.Tree;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
@@ -76,7 +72,6 @@ public class ESDriver extends AbstractLensDriver {
private static final double STREAMING_PARTITION_COST = 0;
private static final QueryCost ES_DRIVER_COST = new FactPartitionBasedQueryCost(STREAMING_PARTITION_COST);
- private Configuration conf;
private ESClient esClient;
private ExecutorService asyncQueryPool;
private ESDriverConfig config;
@@ -89,11 +84,6 @@ public class ESDriver extends AbstractLensDriver {
private final Map<QueryHandle, QueryContext> handleContextMap = Maps.newConcurrentMap();
@Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
public QueryCost estimate(AbstractQueryContext qctx) {
return ES_DRIVER_COST;
}
@@ -232,16 +222,6 @@ public class ESDriver extends AbstractLensDriver {
}
- @Override
- public ImmutableSet<QueryLaunchingConstraint> getQueryConstraints() {
- return ImmutableSet.copyOf(Sets.<QueryLaunchingConstraint>newHashSet());
- }
-
- @Override
- public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
- return ImmutableSet.copyOf(Sets.<WaitingQueriesSelectionPolicy>newHashSet());
- }
-
private ESQuery rewrite(AbstractQueryContext context) throws LensException {
final String key = keyFor(context);
if (rewrittenQueriesCache.containsKey(key)) {
@@ -299,18 +279,15 @@ public class ESDriver extends AbstractLensDriver {
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
super.configure(conf, driverType, driverName);
- this.conf = new Configuration(conf);
- this.conf.addResource("esdriver-default.xml");
- this.conf.addResource(getDriverResourcePath("esdriver-site.xml"));
- config = new ESDriverConfig(this.conf);
+ config = new ESDriverConfig(getConf());
Class klass;
try {
- klass = Class.forName(this.conf.get(ESDriverConfig.CLIENT_CLASS_KEY));
+ klass = Class.forName(getConf().get(ESDriverConfig.CLIENT_CLASS_KEY));
if (klass != null) {
log.debug("Picked up class {}", klass);
if (ESClient.class.isAssignableFrom(klass)) {
final Constructor constructor = klass.getConstructor(ESDriverConfig.class, Configuration.class);
- esClient = (ESClient) constructor.newInstance(config, this.conf);
+ esClient = (ESClient) constructor.newInstance(config, getConf());
log.debug("Successfully instantiated es client of type {}", klass);
}
} else {
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-es/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/resources/hive-site.xml b/lens-driver-es/src/test/resources/hive-site.xml
index e49c876..f2f4b9b 100644
--- a/lens-driver-es/src/test/resources/hive-site.xml
+++ b/lens-driver-es/src/test/resources/hive-site.xml
@@ -50,4 +50,9 @@
<value>true</value>
</property>
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 0d8810f..f307ccc 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -19,7 +19,6 @@
package org.apache.lens.driver.hive;
import static org.apache.lens.server.api.error.LensDriverErrorCode.*;
-import static org.apache.lens.server.api.util.LensUtil.getImplementations;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -45,8 +44,6 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
-import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.cost.QueryCostCalculator;
@@ -70,7 +67,6 @@ import org.apache.hive.service.rpc.thrift.TSessionHandle;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -83,8 +79,6 @@ public class HiveDriver extends AbstractLensDriver {
/** The Constant HIVE_CONNECTION_CLASS. */
public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
- public static final String HIVE_QUERY_HOOK_CLASS = "lens.driver.hive.query.hook.class";
-
/** The Constant HS2_CONNECTION_EXPIRY_DELAY. */
public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.hs2.connection.expiry.delay";
@@ -101,15 +95,6 @@ public class HiveDriver extends AbstractLensDriver {
public static final String HS2_PRIORITY_DEFAULT_RANGES = "VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW";
public static final String SESSION_KEY_DELIMITER = ".";
- public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY
- = "lens.driver.hive.query.launching.constraint.factories";
-
- private static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY
- = "lens.driver.hive.waiting.queries.selection.policy.factories";
-
- /** The driver conf- which will merged with query conf */
- private Configuration driverConf;
-
/** The HiveConf - used for connecting to hive server and metastore */
private HiveConf hiveConf;
@@ -153,17 +138,12 @@ public class HiveDriver extends AbstractLensDriver {
QueryPriorityDecider queryPriorityDecider;
// package-local. Test case can change.
boolean whetherCalculatePriority;
- private DriverQueryHook queryHook;
private static final Map<String, String> SESSION_CONF = new HashMap<String, String>() {
{
put(HiveConf.ConfVars.HIVE_SERVER2_CLOSE_SESSION_ON_DISCONNECT.varname, "false");
}
};
- @Getter
- protected ImmutableSet<QueryLaunchingConstraint> queryConstraints;
- private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies;
-
private String sessionDbKey(String sessionHandle, String database) {
return sessionHandle + SESSION_KEY_DELIMITER + database;
}
@@ -340,11 +320,6 @@ public class HiveDriver extends AbstractLensDriver {
log.info("Hive driver inited");
}
- @Override
- public Configuration getConf() {
- return driverConf;
- }
-
/*
* (non-Javadoc)
*
@@ -353,23 +328,16 @@ public class HiveDriver extends AbstractLensDriver {
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
super.configure(conf, driverType, driverName);
- this.driverConf = new Configuration(conf);
- String driverConfPath = getDriverResourcePath("hivedriver-site.xml");
- this.driverConf.addResource("hivedriver-default.xml");
- this.driverConf.addResource(driverConfPath);
- // resources have to be added separately on hiveConf again because new HiveConf() overrides hive.* properties
- // from HiveConf
this.hiveConf = new HiveConf(conf, HiveDriver.class);
- this.hiveConf.addResource("hivedriver-default.xml");
- this.hiveConf.addResource(driverConfPath);
+ this.hiveConf.addResource(getConf());
- connectionClass = this.driverConf.getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class,
+ connectionClass = getConf().getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class,
ThriftConnection.class);
isEmbedded = (connectionClass.getName().equals(EmbeddedThriftConnection.class.getName()));
- connectionExpiryTimeout = this.driverConf.getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
- whetherCalculatePriority = this.driverConf.getBoolean(HS2_CALCULATE_PRIORITY, true);
- Class<? extends QueryCostCalculator> queryCostCalculatorClass = this.driverConf.getClass(HS2_COST_CALCULATOR,
+ connectionExpiryTimeout = getConf().getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
+ whetherCalculatePriority = getConf().getBoolean(HS2_CALCULATE_PRIORITY, true);
+ Class<? extends QueryCostCalculator> queryCostCalculatorClass = getConf().getClass(HS2_COST_CALCULATOR,
FactPartitionBasedQueryCostCalculator.class, QueryCostCalculator.class);
try {
queryCostCalculator = queryCostCalculatorClass.newInstance();
@@ -377,18 +345,9 @@ public class HiveDriver extends AbstractLensDriver {
throw new LensException("Can't instantiate query cost calculator of class: " + queryCostCalculatorClass, e);
}
queryPriorityDecider = new CostRangePriorityDecider(
- new CostToPriorityRangeConf(driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES))
+ new CostToPriorityRangeConf(getConf().get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES))
);
- try {
- queryHook = driverConf.getClass(
- HIVE_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
- ).newInstance();
- queryHook.setDriver(this);
- } catch (InstantiationException | IllegalAccessException e) {
- throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
- }
- queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, driverConf);
- selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, driverConf);
+
log.info("Hive driver {} configured successfully", getFullyQualifiedName());
}
@@ -809,11 +768,6 @@ public class HiveDriver extends AbstractLensDriver {
}
@Override
- public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
- return selectionPolicies;
- }
-
- @Override
public Priority decidePriority(AbstractQueryContext ctx) {
return decidePriority(ctx, queryPriorityDecider);
}
@@ -1385,9 +1339,4 @@ public class HiveDriver extends AbstractLensDriver {
public boolean hasLensSession(LensSessionHandle session) {
return lensToHiveSession.containsKey(session.getPublicId().toString());
}
-
- @Override
- public DriverQueryHook getQueryHook() {
- return queryHook;
- }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index daf01ac..43b33f3 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -118,7 +118,7 @@ public class TestHiveDriver {
protected void createDriver() throws LensException {
driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driverConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
- driverConf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
+ driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
driverConf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 1acbb13..961ec4e 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -148,7 +148,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driver = new HiveDriver();
driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
- driverConf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
+ driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
driver.configure(driverConf, "hive", "hive1");
drivers = Lists.<LensDriver>newArrayList(driver);
System.out.println("TestRemoteHiveDriver created");
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index e41077c..82e0231 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -23,7 +23,6 @@ import static java.util.Arrays.asList;
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.*;
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.ConnectionPoolProperties.*;
-import static org.apache.lens.server.api.util.LensUtil.getImplementations;
import static com.google.common.base.Preconditions.checkState;
@@ -51,9 +50,7 @@ import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
-import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.rewrite.QueryRewriter;
@@ -68,8 +65,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
-import com.google.common.collect.ImmutableSet;
-
import lombok.*;
import lombok.extern.slf4j.Slf4j;
@@ -95,20 +90,12 @@ public class JDBCDriver extends AbstractLensDriver {
@Getter
private ConcurrentHashMap<QueryHandle, JdbcQueryContext> queryContextMap;
- /** The conf. */
- private Configuration conf;
-
/** Configuration for estimate connection pool */
private Configuration estimateConf;
/** Estimate connection provider */
private ConnectionProvider estimateConnectionProvider;
private LogSegregationContext logSegregationContext;
- private DriverQueryHook queryHook;
-
- @Getter
- private ImmutableSet<QueryLaunchingConstraint> queryConstraints;
- private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies;
private boolean isStatementCancelSupported;
/**
@@ -321,6 +308,7 @@ public class JDBCDriver extends AbstractLensDriver {
queryContext.getLensContext().getDriverStatus().setDriverFinishTime(System.currentTimeMillis());
}
}
+
return result;
}
@@ -383,14 +371,6 @@ public class JDBCDriver extends AbstractLensDriver {
}
}
- /**
- * Get driver configuration
- */
- @Override
- public Configuration getConf() {
- return conf;
- }
-
/*
* (non-Javadoc)
*
@@ -399,18 +379,7 @@ public class JDBCDriver extends AbstractLensDriver {
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
super.configure(conf, driverType, driverName);
- this.conf = new Configuration(conf);
- this.conf.addResource("jdbcdriver-default.xml");
- this.conf.addResource(getDriverResourcePath("jdbcdriver-site.xml"));
- init(conf);
- try {
- queryHook = this.conf.getClass(
- JDBC_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
- ).newInstance();
- queryHook.setDriver(this);
- } catch (InstantiationException | IllegalAccessException e) {
- throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
- }
+ init();
configured = true;
log.info("JDBC Driver {} configured", getFullyQualifiedName());
}
@@ -418,14 +387,12 @@ public class JDBCDriver extends AbstractLensDriver {
/**
* Inits the.
*
- * @param conf the conf
* @throws LensException the lens exception
*/
- protected void init(Configuration conf) throws LensException {
-
- final int maxPoolSize = parseInt(this.conf.get(JDBC_POOL_MAX_SIZE.getConfigKey()));
+ public void init() throws LensException {
+ final int maxPoolSize = parseInt(getConf().get(JDBC_POOL_MAX_SIZE.getConfigKey()));
final int maxConcurrentQueries
- = parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY));
+ = parseInt(getConf().get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY));
checkState(maxPoolSize >= maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:"
+ maxConcurrentQueries);
@@ -439,7 +406,7 @@ public class JDBCDriver extends AbstractLensDriver {
}
});
- Class<? extends ConnectionProvider> cpClass = conf.getClass(JDBC_CONNECTION_PROVIDER,
+ Class<? extends ConnectionProvider> cpClass = getConf().getClass(JDBC_CONNECTION_PROVIDER,
DataSourceConnectionProvider.class, ConnectionProvider.class);
try {
connectionProvider = cpClass.newInstance();
@@ -449,9 +416,8 @@ public class JDBCDriver extends AbstractLensDriver {
throw new LensException(e);
}
this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
- this.queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, this.conf);
- this.selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, this.conf);
- this.isStatementCancelSupported = conf.getBoolean(STATEMENT_CANCEL_SUPPORTED, DEFAULT_STATEMENT_CANCEL_SUPPORTED);
+ this.isStatementCancelSupported = getConf().getBoolean(STATEMENT_CANCEL_SUPPORTED,
+ DEFAULT_STATEMENT_CANCEL_SUPPORTED);
}
/**
@@ -469,7 +435,7 @@ public class JDBCDriver extends AbstractLensDriver {
try {
// Add here to cover the path when the queries are executed it does not
// use the driver conf
- return connectionProvider.getConnection(conf);
+ return connectionProvider.getConnection(getConf());
} catch (SQLException e) {
throw new LensException(e);
}
@@ -483,7 +449,7 @@ public class JDBCDriver extends AbstractLensDriver {
*/
protected QueryRewriter getQueryRewriter() throws LensException {
QueryRewriter rewriter;
- Class<? extends QueryRewriter> queryRewriterClass = conf.getClass(JDBC_QUERY_REWRITER_CLASS,
+ Class<? extends QueryRewriter> queryRewriterClass = getConf().getClass(JDBC_QUERY_REWRITER_CLASS,
DummyQueryRewriter.class, QueryRewriter.class);
try {
rewriter = queryRewriterClass.newInstance();
@@ -492,7 +458,7 @@ public class JDBCDriver extends AbstractLensDriver {
log.error("{} Unable to create rewriter object", getFullyQualifiedName(), e);
throw new LensException(e);
}
- rewriter.init(conf);
+ rewriter.init(getConf());
return rewriter;
}
@@ -674,7 +640,7 @@ public class JDBCDriver extends AbstractLensDriver {
// Get connection config used by estimate pool.
protected final Configuration getEstimateConnectionConf() {
if (estimateConf == null) {
- Configuration tmpConf = new Configuration(conf);
+ Configuration tmpConf = new Configuration(getConf());
// Override JDBC settings in estimate conf, if set by user explicitly. Otherwise fall back to default JDBC pool
// config
for (String key : asList(JDBC_CONNECTION_PROPERTIES, JDBC_DB_URI, JDBC_DRIVER_CLASS, JDBC_USER, JDBC_PASSWORD,
@@ -1059,11 +1025,6 @@ public class JDBCDriver extends AbstractLensDriver {
}
- @Override
- public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
- return this.selectionPolicies;
- }
-
/*
* (non-Javadoc)
*
@@ -1084,12 +1045,6 @@ public class JDBCDriver extends AbstractLensDriver {
public void writeExternal(ObjectOutput arg0) throws IOException {
// TODO Auto-generated method stub
}
-
- @Override
- public DriverQueryHook getQueryHook() {
- return queryHook;
- }
-
@Override
public StatusUpdateMethod getStatusUpdateMethod() {
return StatusUpdateMethod.PUSH;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
index 3c34eb1..f4e0451 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
@@ -38,7 +38,6 @@ public final class JDBCDriverConfConstants {
/** The Constant JDBC_QUERY_REWRITER_CLASS. */
public static final String JDBC_QUERY_REWRITER_CLASS = JDBC_DRIVER_PFX + "query.rewriter";
- public static final String JDBC_QUERY_HOOK_CLASS = JDBC_DRIVER_PFX + "query.hook.class";
/** The Constant JDBC_DRIVER_CLASS. */
public static final String JDBC_DRIVER_CLASS = JDBC_DRIVER_PFX + "driver.class";
@@ -104,11 +103,6 @@ public final class JDBCDriverConfConstants {
public static final String JDBC_FETCH_SIZE = JDBC_DRIVER_PFX + "fetch.size";
public static final int DEFAULT_JDBC_FETCH_SIZE = 1000;
- public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = JDBC_DRIVER_PFX
- + "query.launching.constraint.factories";
-
- public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = JDBC_DRIVER_PFX
- + "waiting.queries.selection.policy.factories";
public static final String REGEX_REPLACEMENT_VALUES = JDBC_DRIVER_PFX + "regex.replacement.values";
public static final String STATEMENT_CANCEL_SUPPORTED = JDBC_DRIVER_PFX + "statement.cancel.supported";
public static final boolean DEFAULT_STATEMENT_CANCEL_SUPPORTED = true;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
index 82b5647..f293ea8 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
@@ -35,14 +35,18 @@ public class MaxJDBCConnectionCheckConstraint implements QueryLaunchingConstrain
}
@Override
- public boolean allowsLaunchOf(final QueryContext candidateQuery,
+ public String allowsLaunchOf(final QueryContext candidateQuery,
EstimatedImmutableQueryCollection launchedQueries) {
final LensDriver selectedDriver = candidateQuery.getSelectedDriver();
- final boolean canLaunch = (selectedDriver instanceof JDBCDriver)
- && (((JDBCDriver) selectedDriver).getQueryContextMap().size() < poolMaxSize);
-
- log.debug("canLaunch:{}", canLaunch);
- return canLaunch;
+ if (!(selectedDriver instanceof JDBCDriver)) {
+ return "driver isn't jdbc driver";
+ }
+ int runningQueries = ((JDBCDriver) selectedDriver).getQueryContextMap().size();
+ if (runningQueries >= poolMaxSize) {
+ return runningQueries + "/" + poolMaxSize + " queries running on driver "
+ + candidateQuery.getSelectedDriver().getFullyQualifiedName();
+ }
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 2ad7f76..a402f91 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -446,11 +446,11 @@ public class TestJdbcDriver {
//new query shouldn't be allowed
QueryContext newcontext = createQueryContext("SELECT 123 FROM max_connection_test");
- assertFalse(constraint.allowsLaunchOf(newcontext, null));
+ assertNotNull(constraint.allowsLaunchOf(newcontext, null));
//close one query and launch the previous query again
driver.closeQuery(context.getQueryHandle());
- assertTrue(constraint.allowsLaunchOf(newcontext, null));
+ assertNull(constraint.allowsLaunchOf(newcontext, null));
close();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
index d80fba9..cff8e91 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
@@ -19,8 +19,8 @@
package org.apache.lens.regression.core.constants;
-import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.driver.jdbc.JDBCDriverConfConstants;
+import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
import org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory;
@@ -38,7 +38,7 @@ public class DriverConfig {
MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY;
public static final String JDBC_POOL_SIZE = JDBCDriverConfConstants.ConnectionPoolProperties.
JDBC_POOL_MAX_SIZE.getConfigKey();
- public static final String HIVE_CONSTRAINT_FACTORIES = HiveDriver.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY;
+ public static final String HIVE_CONSTRAINT_FACTORIES = LensConfConstants.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX;
public static final String MAX_CONCURRENT_CONSTRAINT_FACTORY = MaxConcurrentDriverQueriesConstraintFactory
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 8cf617b..3ae59c6 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -98,7 +98,7 @@ public final class LensConfConstants {
public static final String MAX_SESSIONS_PER_USER = SERVER_PFX + "max.sessions.per.user";
- public static final String QUERY_COMPARATOR_CLASS = SERVER_PFX + "query.comparator.class";
+ public static final String QUERY_COMPARATOR_CLASSES = SERVER_PFX + "query.comparator.classes";
public static final Integer DEFAULT_MAX_SESSIONS_PER_USER = 10;
@@ -913,8 +913,9 @@ public final class LensConfConstants {
/**
* Key to get the implementations of query constraint factories.
*/
+ public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX = "query.launching.constraint.factories";
public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = SERVER_PFX
- + "query.launching.constraint.factories";
+ + QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX;
/**
* Key to get the total query cost ceiling per user.
@@ -925,8 +926,10 @@ public final class LensConfConstants {
/**
* Key to get the implementations of waiting queries selection policy factories.
*/
+ public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX =
+ "waiting.queries.selection.policy.factories";
public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = SERVER_PFX
- + "waiting.queries.selection.policy.factories";
+ + WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX;
/**
* Key denoting the dialect class property of saved query service.
@@ -954,6 +957,18 @@ public final class LensConfConstants {
public static final String DRIVER_WEIGHT = DRIVER_PFX + "weight";
/**
+ * Key for specifying Retry policy class
+ */
+ public static final String RETRY_POLICY_CLASSES_SFX = "query.retry.policy.classes";
+
+ public static final String QUERY_RETRY_POLICY_CLASSES = SERVER_PFX + RETRY_POLICY_CLASSES_SFX;
+
+ /**
+ * Driver hook property
+ */
+ public static final String DRIVER_HOOK_CLASS_SFX = "query.hook.class";
+
+ /**
* Default driver weight
*/
public static final int DEFAULT_DRIVER_WEIGHT = 1;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
deleted file mode 100644
index 17bfba0..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-/**
- * A backoff retry handler.
- *
- * This allows a backoff on any call, so provides methods whether we can try the operation now,
- * whats next time when operation can be performed and whether operation has exhausted all retries.
- *
- * Callers of this would do the following :
- *
- * if (handler.canTryOpNow(FailureContext)) {
- * try {
- * tryCallerOperation();
- * FailureContext.clear();
- * } catch (any Transient Exception) {
- * FailureContext.updateFailure();
- * if (!handler.hasExhaustedRetries(FailureContext)) {
- * // will be tried later again
- * }
- * throw exception;
- * }
- * }
- */
-public interface BackOffRetryHandler {
-
- /**
- * To know whether operation can be done now.
- *
- * @param failContext FailureContext holding failures till now.
- *
- * @return true if operation can be done now, false otherwise.
- */
- boolean canTryOpNow(FailureContext failContext);
-
- /**
- * Get the time when the operation can be done next.
- *
- * @param failContext FailureContext holding failures till now.
- *
- * @return Next operation time in millis since epoch
- */
- long getOperationNextTime(FailureContext failContext);
-
- /**
- * Has the operation exhausted all its retries
- *
- * @param failContext FailureContext holding failures till now.
- *
- * @return true if all retries have exhausted, false otherwise.
- */
- boolean hasExhaustedRetries(FailureContext failContext);
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java
deleted file mode 100644
index 70a34b0..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-import lombok.Getter;
-
-/**
- * Failure context captures last failure time and number of failures.
- */
-public class FailureContext {
-
- @Getter
- private long lastFailedTime = 0;
- @Getter
- private int failCount = 0;
-
- public synchronized void updateFailure() {
- lastFailedTime = System.currentTimeMillis();
- failCount++;
- }
-
- public synchronized void clear() {
- lastFailedTime = 0;
- failCount = 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java
deleted file mode 100644
index e7fb8ce..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-/**
- * A exponential backoff retry handler.
- *
- * It allows the the failures to be retried at a next update time, which can increase exponentially.
- *
- */
-public class FibonacciExponentialBackOffRetryHandler implements BackOffRetryHandler {
- final int[] fibonacci;
- final long maxDelay;
- final long waitMillis;
-
- public FibonacciExponentialBackOffRetryHandler(int numRetries, long maxDelay, long waitMillis) {
- checkArgument(numRetries > 2);
- fibonacci = new int[numRetries];
- fibonacci[0] = 1;
- fibonacci[1] = 1;
- for(int i = 2; i < numRetries; ++i) {
- fibonacci[i] = fibonacci[i-1] + fibonacci[i-2];
- }
- this.maxDelay = maxDelay;
- this.waitMillis = waitMillis;
- }
-
- public boolean canTryOpNow(FailureContext failContext) {
- synchronized (failContext) {
- if (failContext.getFailCount() != 0) {
- long now = System.currentTimeMillis();
- if (now < getOperationNextTime(failContext)) {
- return false;
- }
- }
- return true;
- }
- }
-
- public long getOperationNextTime(FailureContext failContext) {
- synchronized (failContext) {
- if (failContext.getFailCount() >= fibonacci.length) {
- return failContext.getLastFailedTime() + maxDelay;
- }
- long delay = Math.min(maxDelay, fibonacci[failContext.getFailCount()] * waitMillis);
- return failContext.getLastFailedTime() + delay;
- }
- }
-
- public boolean hasExhaustedRetries(FailureContext failContext) {
- synchronized (failContext) {
- if (failContext.getFailCount() >= fibonacci.length) {
- return true;
- }
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java
deleted file mode 100644
index 88fbe43..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-/**
- * Factory which creates operation retry handler
- */
-public class OperationRetryHandlerFactory {
- private OperationRetryHandlerFactory() {
- }
-
- /**
- * Create exponential backoff handler
- *
- * @param numRetries Number of exponential backoff retries
- * @param maxDelay Maximum delay an operation can wait for next
- * @param waitMillis Number of millis that would grow exponentially incase of failures
- *
- * @return BackOffRetryHandler
- */
- public static BackOffRetryHandler createExponentialBackOffHandler(int numRetries, long maxDelay, long waitMillis) {
- return new FibonacciExponentialBackOffRetryHandler(numRetries, maxDelay, waitMillis);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
index 365a619..8f30aa0 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
@@ -18,15 +18,23 @@
*/
package org.apache.lens.server.api.driver;
+import static org.apache.lens.server.api.LensConfConstants.*;
+import static org.apache.lens.server.api.util.LensUtil.getImplementations;
+
import org.apache.lens.api.Priority;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.retry.ChainedRetryPolicyDecider;
+import org.apache.lens.server.api.retry.RetryPolicyDecider;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -47,15 +55,48 @@ public abstract class AbstractLensDriver implements LensDriver {
@Getter
private String fullyQualifiedName = null;
- private DriverQueryHook noOpDriverQueryHook = new NoOpDriverQueryHook();
+ @Getter
+ private Configuration conf;
+
+ @Getter
+ private ImmutableSet<QueryLaunchingConstraint> queryConstraints;
+ @Getter
+ private ImmutableSet<WaitingQueriesSelectionPolicy> waitingQuerySelectionPolicies;
+ @Getter
+ RetryPolicyDecider<QueryContext> retryPolicyDecider;
+ @Getter
+ private DriverQueryHook queryHook;
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
if (StringUtils.isBlank(driverType) || StringUtils.isBlank(driverName)) {
throw new LensException("Driver Type and Name can not be null or empty");
}
- fullyQualifiedName = new StringBuilder(driverType).append(SEPARATOR).append(driverName).toString();
- noOpDriverQueryHook.setDriver(this);
+ fullyQualifiedName = driverType + SEPARATOR + driverName;
+ this.conf = new DriverConfiguration(conf, driverType, getClass());
+ this.conf.addResource(getClass().getSimpleName().toLowerCase() + "-default.xml");
+ this.conf.addResource(getDriverResourcePath(getClass().getSimpleName().toLowerCase() + "-site.xml"));
+
+ this.queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX, getConf());
+ this.waitingQuerySelectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX, getConf());
+
+ loadRetryPolicyDecider();
+ loadQueryHook();
+ }
+
+ protected void loadQueryHook() throws LensException {
+ try {
+ queryHook = getConf().getClass(
+ DRIVER_HOOK_CLASS_SFX, NoOpDriverQueryHook.class, DriverQueryHook.class
+ ).newInstance();
+ queryHook.setDriver(this);
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
+ }
+ }
+
+ protected void loadRetryPolicyDecider() throws LensException {
+ this.retryPolicyDecider = ChainedRetryPolicyDecider.from(getConf(), RETRY_POLICY_CLASSES_SFX);
}
/**
@@ -96,8 +137,8 @@ public abstract class AbstractLensDriver implements LensDriver {
* @return
*/
protected String getDriverResourcePath(String resourceName) {
- return new StringBuilder(LensConfConstants.DRIVERS_BASE_DIR).append(SEPARATOR).append(getFullyQualifiedName())
- .append(SEPARATOR).append(resourceName).toString();
+ return LensConfConstants.DRIVERS_BASE_DIR + SEPARATOR + getFullyQualifiedName()
+ + SEPARATOR + resourceName;
}
@Override
@@ -106,11 +147,6 @@ public abstract class AbstractLensDriver implements LensDriver {
}
@Override
- public DriverQueryHook getQueryHook() {
- return noOpDriverQueryHook;
- }
-
- @Override
public StatusUpdateMethod getStatusUpdateMethod() {
return StatusUpdateMethod.PULL;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
new file mode 100644
index 0000000..69a1a0b
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+
+import static org.apache.lens.server.api.LensConfConstants.DRIVER_PFX;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class DriverConfiguration extends Configuration {
+ private final String driverClassType;
+ private String driverType;
+ private final Class<? extends AbstractLensDriver> driverClass;
+
+ public DriverConfiguration(Configuration conf, String driverType, Class<? extends AbstractLensDriver> driverClass) {
+ super(conf);
+ this.driverType = driverType;
+ this.driverClass = driverClass;
+ this.driverClassType = driverClass.getSimpleName().toLowerCase().replaceAll("driver$", "");
+ }
+
+ @Override
+ public String[] getStrings(String name) {
+ for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name,
+ DRIVER_PFX + name, name, }) {
+ String[] s = super.getStrings(key);
+ if (s != null) {
+ return s;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) {
+ for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name,
+ DRIVER_PFX + name, name, }) {
+ if (getTrimmed(key) != null) {
+ return super.getClass(key, defaultValue, xface);
+ }
+ }
+ return defaultValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
index fc24fc6..b5c5dcd 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
@@ -22,12 +22,12 @@ import java.io.Serializable;
import org.apache.lens.api.query.QueryStatus;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.Data;
/**
* The Class DriverQueryStatus.
*/
+@Data
public class DriverQueryStatus implements Serializable {
/**
@@ -35,6 +35,9 @@ public class DriverQueryStatus implements Serializable {
*/
private static final long serialVersionUID = 1L;
+ public boolean failed() {
+ return state == DriverQueryState.FAILED;
+ }
/**
@@ -96,59 +99,56 @@ public class DriverQueryStatus implements Serializable {
/**
* The progress.
*/
- @Getter
- @Setter
- private double progress = 0.0f;
+ private double progress;
/**
* The state.
*/
- @Getter
- @Setter
- private DriverQueryState state = DriverQueryState.NEW;
+ private DriverQueryState state;
/**
* The status message.
*/
- @Getter
- @Setter
private String statusMessage;
/**
* The is result set available.
*/
- @Getter
- @Setter
- private boolean isResultSetAvailable = false;
+ private boolean isResultSetAvailable;
/**
* The progress message.
*/
- @Getter
- @Setter
private String progressMessage;
/**
* The error message.
*/
- @Getter
- @Setter
private String errorMessage;
/**
* The driver start time.
*/
- @Getter
- @Setter
- private Long driverStartTime = 0L;
+ private Long driverStartTime;
/**
* The driver finish time.
*/
- @Getter
- @Setter
- private Long driverFinishTime = 0L;
+ private Long driverFinishTime;
+ {
+ clear();
+ }
+ public void clear() {
+ progress = 0.0f;
+ state = DriverQueryState.NEW;
+ statusMessage = null;
+ isResultSetAvailable = false;
+ progressMessage = null;
+ errorMessage = null;
+ driverStartTime = 0L;
+ driverFinishTime = 0L;
+ }
/**
* To query status.
*
@@ -169,7 +169,7 @@ public class DriverQueryStatus implements Serializable {
qstate = QueryStatus.Status.EXECUTED;
break;
case FAILED:
- qstate = QueryStatus.Status.FAILED;
+ qstate = QueryStatus.Status.FAILING;
break;
case CANCELED:
qstate = QueryStatus.Status.CANCELED;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index e472de0..1462239 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -31,6 +31,7 @@ import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.retry.RetryPolicyDecider;
import org.apache.hadoop.conf.Configuration;
@@ -232,4 +233,6 @@ public interface LensDriver extends Externalizable {
* @return The method of status update supported by this driver.
*/
StatusUpdateMethod getStatusUpdateMethod();
+
+ RetryPolicyDecider<QueryContext> getRetryPolicyDecider();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index e160f58..c6a872d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -202,7 +202,7 @@ public abstract class AbstractQueryContext implements Serializable {
public Map<LensDriver, DriverEstimateRunnable> getDriverEstimateRunnables() throws LensException {
Map<LensDriver, DriverEstimateRunnable> estimateRunnables = new HashMap<LensDriver, DriverEstimateRunnable>();
- for (LensDriver driver : driverContext.getDrivers()) {
+ for (LensDriver driver : driverContext.getEligibleDrivers()) {
estimateRunnables.put(driver, new DriverEstimateRunnable(this, driver));
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 5ff59bd..8e431d1 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -50,8 +50,8 @@ public class DriverSelectorQueryContext {
*/
@Getter
@Setter
- protected Map<LensDriver, DriverQueryContext> driverQueryContextMap = new HashMap<LensDriver,
- DriverQueryContext>();
+ protected Map<LensDriver, DriverQueryContext> driverQueryContextMap = new HashMap<>();
+ private Set<LensDriver> blackListedDrivers = Sets.newHashSet();
public DriverSelectorQueryContext(final String userQuery, final Configuration queryConf,
final Collection<LensDriver> drivers) {
@@ -77,6 +77,10 @@ public class DriverSelectorQueryContext {
}
}
+ public void blacklist(LensDriver selectedDriver) {
+ blackListedDrivers.add(selectedDriver);
+ }
+
public static class DriverQueryContext {
@Getter
@@ -264,13 +268,16 @@ public class DriverSelectorQueryContext {
public Collection<LensDriver> getDrivers() {
return driverQueryContextMap.keySet();
}
+ public Collection<LensDriver> getEligibleDrivers() {
+ return Sets.difference(driverQueryContextMap.keySet(), blackListedDrivers);
+ }
public Collection<LensDriver> getDriversWithValidQueryCost() {
final Set<LensDriver> eligibleDrivers = Sets.newLinkedHashSet();
- for (Map.Entry<LensDriver, DriverQueryContext> driverToDriverContext : this.driverQueryContextMap.entrySet()) {
- if (driverToDriverContext.getValue().driverCost != null) {
- eligibleDrivers.add(driverToDriverContext.getKey());
+ for (LensDriver driver: getEligibleDrivers()) {
+ if (driverQueryContextMap.get(driver).driverCost != null) {
+ eligibleDrivers.add(driver);
}
}
return Collections.unmodifiableCollection(eligibleDrivers);
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index b58fcf9..d88944b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -19,10 +19,11 @@
package org.apache.lens.server.api.query;
import java.util.Collection;
-import java.util.Iterator;
+import java.util.List;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.Priority;
+import org.apache.lens.api.query.FailedAttempt;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.driver.LensDriver;
@@ -46,7 +47,7 @@ import lombok.ToString;
*
* @see java.lang.Object#hashCode()
*/
-@EqualsAndHashCode(exclude = {"selectedDriver", "conf"})
+@EqualsAndHashCode(exclude = {"selectedDriver", "conf", "failedAttempts"})
/*
* (non-Javadoc)
*
@@ -185,6 +186,10 @@ public class FinishedLensQuery {
@Setter
private String driverQuery;
+ @Getter
+ @Setter
+ private List<FailedAttempt> failedAttempts;
+
/**
* Instantiates a new finished lens query.
*/
@@ -222,6 +227,7 @@ public class FinishedLensQuery {
this.priority = ctx.getPriority().toString();
}
this.conf = ctx.getLensConf();
+ this.failedAttempts = ctx.getFailedAttempts();
}
public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) {
@@ -249,13 +255,12 @@ public class FinishedLensQuery {
if (getPriority() != null) {
qctx.setPriority(Priority.valueOf(getPriority()));
}
+ qctx.setFailedAttempts(getFailedAttempts());
return qctx;
}
private LensDriver getDriverFromName(Collection<LensDriver> drivers) {
- Iterator<LensDriver> iterator = drivers.iterator();
- while (iterator.hasNext()) {
- LensDriver driver = iterator.next();
+ for (LensDriver driver : drivers) {
if (driverName.equals(driver.getFullyQualifiedName())) {
return driver;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java
deleted file mode 100644
index eaf3fee..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.Priority;
-import org.apache.lens.api.query.QueryHandle;
-
-/**
- * Event fired when query priority changes.
- */
-public class PriorityChange extends QueryEvent<Priority> {
-
- /**
- * Instantiates a new priority change.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public PriorityChange(long eventTime, Priority prev, Priority current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java
deleted file mode 100644
index 8980b61..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-
-/**
- * The Class QueryAccepted.
- */
-public class QueryAccepted extends QueryEvent<String> {
-
- /**
- * Instantiates a new query accepted.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueryAccepted(long eventTime, String prev, String current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java
deleted file mode 100644
index a473a47..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when query is cancelled.
- */
-public class QueryCancelled extends QueryEnded {
-
- /**
- * Instantiates a new query cancelled.
- *
- * @param ctx the query context
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- * @param user the user
- * @param cause the cause
- */
- public QueryCancelled(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
- QueryHandle handle,
- String user, String cause) {
- super(ctx, eventTime, prev, current, handle, user, cause);
- checkCurrentState(QueryStatus.Status.CANCELED);
- }
-
- public QueryCancelled(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
- // TODO: correct username. put who cancelled it, not the submitter. Similar for others
- this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java
deleted file mode 100644
index 3837087..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when a query is closed.
- */
-public class QueryClosed extends QueryEnded {
-
- /**
- * Instantiates a new query closed.
- *
- * @param ctx the query context
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- * @param user the user
- * @param cause the cause
- */
- public QueryClosed(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
- QueryHandle handle,
- String user, String cause) {
- super(ctx, eventTime, prev, current, handle, user, cause);
- checkCurrentState(QueryStatus.Status.CLOSED);
- }
-
- public QueryClosed(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
- this(ctx, ctx.getClosedTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
- }
-}