You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2016/10/17 05:45:14 UTC
[1/4] lens git commit: LENS-743: Query retry framework for retrying
upon transient failures
Repository: lens
Updated Branches:
refs/heads/master 182f6dcac -> 38ab6c608
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
index c683a2c..00130d0 100644
--- a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerServiceImpl.java
@@ -41,8 +41,8 @@ import org.apache.lens.server.api.LensErrorInfo;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.SchedulerAlarmEvent;
import org.apache.lens.server.api.health.HealthStatus;
-import org.apache.lens.server.api.query.QueryEnded;
import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.api.query.events.QueryEnded;
import org.apache.lens.server.api.scheduler.SchedulerService;
import org.apache.lens.server.error.LensSchedulerErrorCode;
import org.apache.lens.server.session.LensSessionImpl;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
index 46adb7b..20243f4 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryContextComparatorTest.java
@@ -27,16 +27,26 @@ import java.util.Comparator;
import org.apache.lens.api.Priority;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.comparators.ChainedComparator;
+import org.apache.lens.server.api.query.comparators.FIFOQueryComparator;
+import org.apache.lens.server.api.query.comparators.QueryCostComparator;
+import org.apache.lens.server.api.query.comparators.QueryPriorityComparator;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
+import com.google.common.collect.Lists;
+
public class QueryContextComparatorTest {
private final Comparator<QueryContext> priorityComparator = new QueryPriorityComparator();
private final Comparator<QueryContext> costComparator = new QueryCostComparator();
-
+ private final Comparator<QueryContext> fifoComparator = new FIFOQueryComparator();
+ private final Comparator<QueryContext> priorityAndFifoComparator
+ = new ChainedComparator<>(Lists.newArrayList(priorityComparator, fifoComparator));
+ private final Comparator<QueryContext> costAndFifoComparator
+ = new ChainedComparator<>(Lists.newArrayList(costComparator, fifoComparator));
@DataProvider
@@ -61,7 +71,7 @@ public class QueryContextComparatorTest {
when(query2.getSelectedDriverQueryCost()).thenReturn(qcO2);
when(qcO1.compareTo(qcO2)).thenReturn(resultOfQueryCostCompare);
- assertEquals(costComparator.compare(query1, query2), expectedResult);
+ assertEquals(costAndFifoComparator.compare(query1, query2), expectedResult);
}
@Test
@@ -73,7 +83,7 @@ public class QueryContextComparatorTest {
QueryContext query2 = mock(QueryContext.class);
when(query2.getPriority()).thenReturn(Priority.LOW); // Ordinal = 3
- assertEquals(priorityComparator.compare(query1, query2), -2);
+ assertEquals(priorityAndFifoComparator.compare(query1, query2), -2);
}
@@ -114,8 +124,8 @@ public class QueryContextComparatorTest {
// Cost and Priority both are same, hence the comparison should happen
// on query submission time
- assertEquals(priorityComparator.compare(query1, query2), expectedResult);
- assertEquals(costComparator.compare(query1, query2), expectedResult);
+ assertEquals(priorityAndFifoComparator.compare(query1, query2), expectedResult);
+ assertEquals(costAndFifoComparator.compare(query1, query2), expectedResult);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
index 526accc..b906776 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestEventService.java
@@ -39,7 +39,7 @@ import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.events.LensEvent;
import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.events.LensEventService;
-import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.events.*;
import org.apache.lens.server.api.session.SessionClosed;
import org.apache.lens.server.api.session.SessionExpired;
import org.apache.lens.server.api.session.SessionOpened;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
index e620dc5..066525b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
@@ -25,20 +25,21 @@ import java.sql.Statement;
import java.util.HashMap;
import java.util.List;
+import javax.ws.rs.core.Application;
+
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.Priority;
+import org.apache.lens.api.query.FailedAttempt;
import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.driver.jdbc.JDBCResultSet;
+import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.driver.MockDriver;
-import org.apache.lens.server.api.query.DriverSelectorQueryContext;
-import org.apache.lens.server.api.query.FinishedLensQuery;
-import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.api.query.*;
import org.apache.hadoop.conf.Configuration;
@@ -54,7 +55,12 @@ import lombok.extern.slf4j.Slf4j;
*/
@Test(groups = "unit-test")
@Slf4j
-public class TestLensDAO {
+public class TestLensDAO extends LensJerseyTest {
+
+ @Override
+ protected Application configure() {
+ return new TestQueryService.QueryServiceTestApp();
+ }
/**
* Test lens server dao.
@@ -89,6 +95,10 @@ public class TestLensDAO {
finishedLensQuery.setStatus(QueryStatus.Status.SUCCESSFUL.name());
finishedLensQuery.setPriority(Priority.NORMAL.toString());
+ finishedLensQuery.setFailedAttempts(Lists.newArrayList(
+ new FailedAttempt("driver1", 1.0, "progress full", "no error", 0L, 1L),
+ new FailedAttempt("driver2", 1.0, "progress also full", "no error at all", 2L, 3L)
+ ));
// Validate JDBC driver RS Meta can be deserialized
// Create a valid JDBCResultSet
@@ -145,7 +155,8 @@ public class TestLensDAO {
Assert.assertEquals(Priority.valueOf(actual.getPriority()), Priority.NORMAL);
Assert.assertEquals(actual.getDriverQuery(), driverQuery);
// when driver list contains the selected driver, selected driver should get set correctly in context
- QueryContext retrievedQueryContext = actual.toQueryContext(new Configuration(), Lists.newArrayList(mockDriver));
+ QueryContext retrievedQueryContext = actual.toQueryContext(new Configuration(),
+ Lists.newArrayList(mockDriver));
Assert.assertEquals(retrievedQueryContext.getSelectedDriverQuery(), driverQuery);
// Test find finished queries
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
index bf4577c..8c1bb7b 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryIndependenceFromSessionClose.java
@@ -102,6 +102,7 @@ public class TestQueryIndependenceFromSessionClose extends LensJerseyTest {
*/
@BeforeClass
public void setUpClass() throws Exception {
+ restartLensServer(getServerConf());
lensSessionId = getSession();
createTable(TEST_TABLE);
loadData(TEST_TABLE, TestResourceFile.TEST_DATA2_FILE.getValue());
@@ -110,6 +111,10 @@ public class TestQueryIndependenceFromSessionClose extends LensJerseyTest {
QUERY_PERSISTENT_RESULT_INDRIVER, true,
QUERY_OUTPUT_FORMATTER, TestQueryService.DeferredPersistentResultFormatter.class.getName());
}
+ @AfterClass
+ public void restart() {
+ restartLensServer();
+ }
@Override
public Map<String, String> getServerConfOverWrites() {
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
index 3a3f2f1..6a57b20 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryNotifictaionResource.java
@@ -27,6 +27,7 @@ import org.apache.lens.server.api.error.LensException;
import org.glassfish.jersey.media.multipart.FormDataParam;
+import org.testng.Assert;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -59,7 +60,8 @@ public class TestQueryNotifictaionResource {
+ " status:" + query.getStatus() + " eventtype:" + eventtype);
finishedCount++;
-
+ Assert.assertTrue(query.getQueryName().toUpperCase().contains(query.getStatus().getStatus().name()),
+ "query " + query.getQueryName() + " " + query.getStatus());
if (query.getStatus().successful()) {
successfulCount++;
} else if (query.getStatus().failed()) {
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 440c30b..415f56f 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -1919,35 +1919,49 @@ public class TestQueryService extends LensJerseyTest {
@Test(dataProvider = "mediaTypeData")
- public void testFinishedNotifictaion(MediaType mt) throws LensException, InterruptedException {
- String query = "select ID, IDSTR, count(*) from " + TEST_TABLE + " group by ID, IDSTR";
- String endpoint = getBaseUri() + "/queryapi/notifictaion/finished";
- LensConf conf = new LensConf();
- conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_TYPE_FINISHED, "true");
- conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_MEDIATYPE, mt);
- conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_URLS, endpoint + " , " + endpoint);
-
- //Test for SUCCESSFUL FINISH notification
- queryService.execute(lensSessionId, query, 20000, conf, "testHttpNotifictaionQuery");
-
- //TEST for CANCELLED FINISH notification
- conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true");
- conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, DeferredPersistentResultFormatter.class.getName());
- conf.addProperty("deferPersistenceByMillis", 5000); // defer persistence for 5 secs
- QueryHandle hanlde = queryService.executeAsync(lensSessionId, query, conf, "testHttpNotifictaionQuery");
- queryService.cancelQuery(lensSessionId, hanlde);
-
- //Test for FAILED FINISH notification
- conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, "wrong.formatter");
- queryService.execute(lensSessionId, query, 20000, conf, "testHttpNotifictaionQuery");
-
- Thread.sleep(3000); //Keep some time for notifications to get delivered
- assertEquals(TestQueryNotifictaionResource.getSuccessfulCount(), 2);
- assertEquals(TestQueryNotifictaionResource.getCancelledCount(), 2);
- assertEquals(TestQueryNotifictaionResource.getFailedCount(), 2);
- assertEquals(TestQueryNotifictaionResource.getFinishedCount(), 6);
-
- TestQueryNotifictaionResource.clearState();
+ public void testFinishedNotification(MediaType mt) throws LensException, InterruptedException {
+ try {
+ String query = "select ID, IDSTR, count(*) from " + TEST_TABLE + " group by ID, IDSTR";
+ String endpoint = getBaseUri() + "/queryapi/notifictaion/finished";
+ LensConf conf = new LensConf();
+ conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_TYPE_FINISHED, "true");
+ conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_MEDIATYPE, mt);
+ conf.addProperty(LensConfConstants.QUERY_HTTP_NOTIFICATION_URLS, endpoint + " , " + endpoint);
+
+ //Test for SUCCESSFUL FINISH notification
+ QueryHandle handle1 = queryService.executeAsync(lensSessionId, query, conf,
+ "testHttpNotificationQuerySuccessful");
+
+ //TEST for CANCELLED FINISH notification
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true");
+ conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, DeferredPersistentResultFormatter.class.getName());
+ conf.addProperty("deferPersistenceByMillis", 5000); // defer persistence for 5 secs
+ QueryHandle handle2 = queryService.executeAsync(lensSessionId, query, conf, "testHttpNotificationQueryCanceled");
+ queryService.cancelQuery(lensSessionId, handle2);
+
+ //Test for FAILED FINISH notification
+ conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, "wrong.formatter");
+ QueryHandle handle3 = queryService.executeAsync(lensSessionId, query, conf, "testHttpNotificationQueryFailed");
+
+ for (QueryHandle handle : new QueryHandle[]{handle1, handle2, handle3}) {
+ LensQuery lensQuery = queryService.getQuery(lensSessionId, handle);
+ while (!lensQuery.getStatus().finished()) {
+ Thread.sleep(1000);
+ lensQuery = queryService.getQuery(lensSessionId, handle);
+ }
+ assertTrue(lensQuery.getQueryName().toUpperCase().contains(lensQuery.getStatus().getStatus().name()),
+ "Query finished with wrong status: " + lensQuery);
+ log.info("query {} finished", lensQuery);
+ }
+ // sleep more to allow notifications to go
+ Thread.sleep(3000);
+ assertEquals(TestQueryNotifictaionResource.getFinishedCount(), 6);
+ assertEquals(TestQueryNotifictaionResource.getSuccessfulCount(), 2);
+ assertEquals(TestQueryNotifictaionResource.getCancelledCount(), 2);
+ assertEquals(TestQueryNotifictaionResource.getFailedCount(), 2);
+ } finally {
+ TestQueryNotifictaionResource.clearState();
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java b/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java
index 8b72c81..82fd8ff 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/collect/QueryCollectUtil.java
@@ -25,9 +25,9 @@ import static java.lang.reflect.Modifier.isSynchronized;
import org.apache.lens.api.Priority;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.comparators.QueryCostComparator;
+import org.apache.lens.server.api.query.comparators.QueryPriorityComparator;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
-import org.apache.lens.server.query.QueryCostComparator;
-import org.apache.lens.server.query.QueryPriorityComparator;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java
index ab030cc..9bb159d 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsCheckerTest.java
@@ -23,6 +23,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.testng.Assert.assertEquals;
+import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
@@ -45,24 +46,27 @@ public class DefaultQueryLaunchingConstraintsCheckerTest {
final QueryLaunchingConstraint constraint2 = mock(QueryLaunchingConstraint.class);
QueryLaunchingConstraintsChecker constraintsChecker
= new DefaultQueryLaunchingConstraintsChecker(ImmutableSet.of(constraint1, constraint2));
-
+ QueryStatus status = QueryStatus.getQueuedStatus();
final QueryLaunchingConstraint driverConstraint = mock(QueryLaunchingConstraint.class);
when(mockCandidateQuery.getSelectedDriverQueryConstraints()).thenReturn(ImmutableSet.of(driverConstraint));
+ when(mockCandidateQuery.getStatus()).thenReturn(status);
/* Constraint1 stubbed to pass */
- when(constraint1.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
+ when(constraint1.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(null);
/* Constraint2 stubbed to fail */
- when(constraint2.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(false);
+ when(constraint2.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn("constraint 2 failed");
/* DriverConstraint stubbed to fail */
- when(driverConstraint.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(false);
+ when(driverConstraint.allowsLaunchOf(mockCandidateQuery, mockRunningQueries))
+ .thenReturn("driver constraint failed");
/* Execute test */
boolean canLaunchQuery = constraintsChecker.canLaunch(mockCandidateQuery, mockRunningQueries);
/* Verify */
Assert.assertFalse(canLaunchQuery);
+ Assert.assertEquals(mockCandidateQuery.getStatus().getProgressMessage(), "constraint 2 failed");
}
@Test
@@ -81,9 +85,9 @@ public class DefaultQueryLaunchingConstraintsCheckerTest {
when(mockCandidateQuery.getSelectedDriverQueryConstraints()).thenReturn(ImmutableSet.of(driverConstraint));
/* all constraints stubbed to pass */
- when(constraint1.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
- when(constraint2.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
- when(driverConstraint.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(true);
+ when(constraint1.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(null);
+ when(constraint2.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(null);
+ when(driverConstraint.allowsLaunchOf(mockCandidateQuery, mockRunningQueries)).thenReturn(null);
/* Execute test */
boolean canLaunchQuery = constraintsChecker.canLaunch(mockCandidateQuery, mockRunningQueries);
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java
index d972780..97a7a59 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/ThreadSafeEstimatedQueryCollectionTest.java
@@ -21,7 +21,8 @@ package org.apache.lens.server.query.constraint;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.query.QueryContext;
@@ -74,8 +75,12 @@ public class ThreadSafeEstimatedQueryCollectionTest {
QueryContext mockCandidateQuery = mock(QueryContext.class);
when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver);
when(mockCandidateQuery.getSelectedDriverQueryCost()).thenReturn(COST);
- boolean actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, col);
+ String actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, col);
- assertEquals(actualCanLaunch, expectedCanLaunch);
+ if (expectedCanLaunch) {
+ assertNull(actualCanLaunch);
+ } else {
+ assertNotNull(actualCanLaunch);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java b/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java
index 460190a..80229e8 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraintTest.java
@@ -21,7 +21,8 @@ package org.apache.lens.server.query.constraint;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
@@ -42,7 +43,7 @@ public class TotalQueryCostCeilingConstraintTest {
}
@Test(dataProvider = "dpTestAllowsLaunchOfQuery")
- public void testAllowsLaunchOfQuery(final double totalQueryCostForCurrentUser, final boolean expectedCanLaunchQuery) {
+ public void testAllowsLaunchOfQuery(final double totalQueryCostForCurrentUser, final boolean expectedCanLaunch) {
final QueryCost totalQueryCostCeilingPerUser = new FactPartitionBasedQueryCost(90.0);
final QueryLaunchingConstraint queryConstraint
@@ -56,8 +57,13 @@ public class TotalQueryCostCeilingConstraintTest {
when(launchedQueries.getTotalQueryCost(mockUser))
.thenReturn(new FactPartitionBasedQueryCost(totalQueryCostForCurrentUser));
- boolean actualCanLaunchQuery = queryConstraint.allowsLaunchOf(query, launchedQueries);
- assertEquals(actualCanLaunchQuery, expectedCanLaunchQuery);
+ String actualCanLaunch = queryConstraint.allowsLaunchOf(query, launchedQueries);
+
+ if (expectedCanLaunch) {
+ assertNull(actualCanLaunch);
+ } else {
+ assertNotNull(actualCanLaunch);
+ }
}
@Test
@@ -69,7 +75,7 @@ public class TotalQueryCostCeilingConstraintTest {
final QueryContext query = mock(QueryContext.class);
final EstimatedImmutableQueryCollection launchedQueries = mock(EstimatedImmutableQueryCollection.class);
- boolean actualCanLaunchQuery = queryConstraint.allowsLaunchOf(query, launchedQueries);
- assertEquals(actualCanLaunchQuery, true);
+ String actualCanLaunch = queryConstraint.allowsLaunchOf(query, launchedQueries);
+ assertNull(actualCanLaunch);
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/retry/MockDriverForRetries.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/retry/MockDriverForRetries.java b/lens-server/src/test/java/org/apache/lens/server/query/retry/MockDriverForRetries.java
new file mode 100644
index 0000000..f9cf6c8
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/retry/MockDriverForRetries.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.retry;
+
+import org.apache.lens.server.api.driver.DriverQueryStatus;
+import org.apache.lens.server.api.driver.MockDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+import org.apache.hadoop.conf.Configuration;
+
+import lombok.Getter;
+
+
+public class MockDriverForRetries extends MockDriver {
+ private int numRetries;
+ @Getter
+ private String fullyQualifiedName;
+
+ @Override
+ public void configure(Configuration conf, String driverType, String driverName) throws LensException {
+ this.conf = conf;
+ this.fullyQualifiedName = driverType + "/" + driverName;
+ this.conf.addResource(getDriverResourcePath("driver-site.xml"));
+ this.numRetries = this.conf.getInt("num.retries", 0);
+ loadQueryHook();
+ loadRetryPolicyDecider();
+ }
+
+ private String getDriverProperty(QueryContext ctx, String name) {
+ return ctx.getLensConf().getProperty("driver." + this.getFullyQualifiedName() + "." + name);
+ }
+
+ @Override
+ public void updateStatus(QueryContext context) throws LensException {
+ if (context.getFailedAttempts().size() < numRetries) {
+ String errorMessage = getDriverProperty(context, "error.message");
+ if (errorMessage == null) {
+ errorMessage = "Simulated Failure";
+ }
+ context.getDriverStatus().setState(DriverQueryStatus.DriverQueryState.FAILED);
+ context.getDriverStatus().setErrorMessage(errorMessage);
+ } else {
+ context.getDriverStatus().setState(DriverQueryStatus.DriverQueryState.SUCCESSFUL);
+ }
+ context.getDriverStatus().setDriverFinishTime(System.currentTimeMillis());
+ }
+
+ @Override
+ public void executeAsync(QueryContext context) throws LensException {
+ super.executeAsync(context);
+ context.getDriverStatus().setDriverStartTime(System.currentTimeMillis());
+ }
+
+ @Override
+ public QueryCost estimate(AbstractQueryContext qctx) throws LensException {
+ String cost = qctx.getLensConf().getProperty("driver." + this.getFullyQualifiedName() + ".cost");
+ if (cost == null) {
+ throw new LensException("Can't run query");
+ }
+ return new FactPartitionBasedQueryCost(Double.parseDouble(cost));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/retry/MockRetryPolicyDecider.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/retry/MockRetryPolicyDecider.java b/lens-server/src/test/java/org/apache/lens/server/query/retry/MockRetryPolicyDecider.java
new file mode 100644
index 0000000..e65e191
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/retry/MockRetryPolicyDecider.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.retry;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.retry.BackOffRetryHandler;
+import org.apache.lens.server.api.retry.FibonacciExponentialBackOffRetryHandler;
+import org.apache.lens.server.api.retry.ImmediateRetryHandler;
+import org.apache.lens.server.api.retry.RetryPolicyDecider;
+
+public class MockRetryPolicyDecider implements RetryPolicyDecider<QueryContext> {
+ @Override
+ public BackOffRetryHandler<QueryContext> decidePolicy(String errorMessage) {
+ if (errorMessage.contains("fibonacci")) {
+ return new FibonacciExponentialBackOffRetryHandler<>(3, 10000, Long.parseLong(errorMessage.substring(10)));
+ }
+ return new ImmediateRetryHandler<>(1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/retry/QueryRetryTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/retry/QueryRetryTest.java b/lens-server/src/test/java/org/apache/lens/server/query/retry/QueryRetryTest.java
new file mode 100644
index 0000000..b90b043
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/retry/QueryRetryTest.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.retry;
+
+
+import static org.apache.lens.server.api.LensConfConstants.DRIVER_TYPES_AND_CLASSES;
+import static org.apache.lens.server.api.LensConfConstants.QUERY_RETRY_POLICY_CLASSES;
+import static org.apache.lens.server.api.LensServerAPITestUtil.getLensConf;
+import static org.apache.lens.server.api.util.LensUtil.getHashMap;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+import java.util.Map;
+
+import javax.ws.rs.core.Application;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.lens.api.LensSessionHandle;
+import org.apache.lens.api.query.FailedAttempt;
+import org.apache.lens.api.query.LensQuery;
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.server.LensJerseyTest;
+import org.apache.lens.server.LensServices;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.api.session.SessionService;
+import org.apache.lens.server.common.RestAPITestUtil;
+import org.apache.lens.server.query.QueryExecutionServiceImpl;
+import org.apache.lens.server.query.TestQueryService;
+
+import org.glassfish.jersey.test.TestProperties;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+@Test(groups = "post-restart", dependsOnGroups = "restart-test")
+public class QueryRetryTest extends LensJerseyTest {
+
+ private QueryExecutionServiceImpl queryService;
+ private SessionService sessionService;
+ private LensSessionHandle session;
+
+ private QueryExecutionServiceImpl getQueryService() {
+ return queryService = LensServices.get().getService(QueryExecutionService.NAME);
+ }
+
+ @Override
+ protected Application configure() {
+ enable(TestProperties.LOG_TRAFFIC);
+ enable(TestProperties.DUMP_ENTITY);
+ return new TestQueryService.QueryServiceTestApp();
+ }
+
+ private SessionService getSessionService() {
+ return sessionService = LensServices.get().getService(SessionService.NAME);
+ }
+
+ private LensSessionHandle getSession() throws LensException {
+ return getSessionService().openSession("foo", "bar", null, null);
+ }
+
+ @BeforeTest
+ public void setUp() throws Exception {
+ super.setUp();
+ }
+ @BeforeClass
+ public void setUpClass() throws Exception {
+ restartLensServer(getServerConf(), false);
+ session = getSession();
+ }
+ @AfterClass
+ public void cleanupClass() throws Exception {
+ getSessionService().closeSession(session);
+ restartLensServer();
+ }
+
+ @Override
+ public Map<String, String> getServerConfOverWrites() {
+ return getHashMap(DRIVER_TYPES_AND_CLASSES, "retry:org.apache.lens.server.query.retry.MockDriverForRetries",
+ QUERY_RETRY_POLICY_CLASSES, TestServerRetryPolicyDecider.class.getName());
+ }
+
+ @Test
+ public void testSingleRetrySameDriver() throws LensException, InterruptedException {
+ QueryHandle handle = getQueryService().executeAsync(session, "select 1",
+ getLensConf("driver.retry/single_failure.cost", "1", "driver.retry/double_failure.cost", "2"),
+ "random query");
+ QueryContext ctx = getQueryService().getQueryContext(handle);
+ while (!ctx.getStatus().finished()) {
+ ctx = getQueryService().getQueryContext(handle);
+ Thread.sleep(1000);
+ }
+ assertEquals(ctx.getFailedAttempts().size(), 1);
+ FailedAttempt failedAttempt = ctx.getFailedAttempts().get(0);
+ assertEquals(failedAttempt.getDriverName(), "retry/single_failure");
+ assertEquals(ctx.getSelectedDriver().getFullyQualifiedName(), "retry/single_failure");
+ assertTrue(failedAttempt.getDriverFinishTime() > failedAttempt.getDriverStartTime());
+ assertTrue(ctx.getDriverStatus().getDriverStartTime() > failedAttempt.getDriverFinishTime());
+
+ }
+
+ @Test
+ public void testRetryOnDifferentDriver() throws LensException, InterruptedException {
+ QueryHandle handle = getQueryService().executeAsync(session, "select 1",
+ getLensConf("driver.retry/single_failure.cost", "2", "driver.retry/double_failure.cost", "1"),
+ "random query");
+ QueryContext ctx = getQueryService().getQueryContext(handle);
+ while (!ctx.getStatus().finished()) {
+ ctx = getQueryService().getQueryContext(handle);
+ Thread.sleep(1000);
+ }
+ assertEquals(ctx.getFailedAttempts().size(), 2);
+ FailedAttempt attempt1 = ctx.getFailedAttempts().get(0);
+ FailedAttempt attempt2 = ctx.getFailedAttempts().get(1);
+ // two retries on double_failure
+ assertEquals(attempt1.getDriverName(), "retry/double_failure");
+ assertEquals(attempt2.getDriverName(), "retry/double_failure");
+ // first retry failed on single_failure since the driver checks total retries before failing.
+ // If there weren't any attempts before this one, then this attempt would fail, but since there
+ // have already been 2 attempts and 2 > 1, this attempt passed.
+ assertEquals(ctx.getSelectedDriver().getFullyQualifiedName(), "retry/single_failure");
+ assertTrue(attempt2.getDriverStartTime() > attempt1.getDriverFinishTime());
+ assertTrue(ctx.getDriverStatus().getDriverStartTime() > attempt2.getDriverFinishTime());
+
+ // test rest api
+ LensQuery lensQuery = RestAPITestUtil.getLensQuery(target(), session, handle, MediaType.APPLICATION_XML_TYPE);
+ assertEquals(lensQuery.getFailedAttempts(), ctx.getFailedAttempts());
+ }
+
+ @Test
+ public void testFailureAfterRetry() throws LensException, InterruptedException {
+ QueryHandle handle = getQueryService().executeAsync(session, "select 1",
+ getLensConf("driver.retry/double_failure.cost", "1"),
+ "random query");
+ QueryContext ctx = getQueryService().getQueryContext(handle);
+ while (!ctx.getStatus().finished()) {
+ ctx = getQueryService().getQueryContext(handle);
+ Thread.sleep(1000);
+ }
+ assertTrue(ctx.getStatus().failed());
+ assertEquals(ctx.getFailedAttempts().size(), 1);
+ FailedAttempt attempt1 = ctx.getFailedAttempts().get(0);
+ assertEquals(attempt1.getDriverName(), "retry/double_failure");
+ assertEquals(ctx.getSelectedDriver().getFullyQualifiedName(), "retry/double_failure");
+ assertTrue(ctx.getStatus().failed());
+ }
+
+ @Test
+ public void testDelayedLaunch() throws LensException, InterruptedException {
+ QueryHandle handle = getQueryService().executeAsync(session, "select 1",
+ getLensConf("driver.retry/double_failure.cost", "1",
+ "driver.retry/double_failure.error.message", "fibonacci.500"),
+ "random query");
+ QueryContext ctx = getQueryService().getQueryContext(handle);
+
+ while (!ctx.getStatus().finished()) {
+ ctx = getQueryService().getQueryContext(handle);
+ Thread.sleep(1000);
+ }
+ assertTrue(ctx.getStatus().successful());
+ assertEquals(ctx.getFailedAttempts().size(), 2);
+ FailedAttempt attempt1 = ctx.getFailedAttempts().get(0);
+ FailedAttempt attempt2 = ctx.getFailedAttempts().get(1);
+ assertTrue(attempt2.getDriverStartTime() - attempt1.getDriverFinishTime() >= 500);
+ assertTrue(ctx.getDriverStatus().getDriverStartTime() - attempt2.getDriverFinishTime() >= 1000);
+ }
+
+ @Test
+ public void testRestartWhileRetry() throws LensException, InterruptedException {
+ QueryHandle handle = getQueryService().executeAsync(session, "select 1",
+ getLensConf("driver.retry/double_failure.cost", "1",
+ "driver.retry/double_failure.error.message", "fibonacci.5000"),
+ "random query");
+ QueryContext ctx = getQueryService().getQueryContext(handle);
+
+ while (ctx.getFailedAttempts().size() == 0) {
+ ctx = getQueryService().getQueryContext(handle);
+ Thread.sleep(1000);
+ }
+ restartLensServer(getServerConf(), false);
+ ctx = getQueryService().getQueryContext(handle);
+ while (!ctx.getStatus().finished()) {
+ ctx = getQueryService().getQueryContext(handle);
+ Thread.sleep(1000);
+ }
+ assertTrue(ctx.getStatus().successful());
+ assertEquals(ctx.getFailedAttempts().size(), 2);
+ FailedAttempt attempt1 = ctx.getFailedAttempts().get(0);
+ FailedAttempt attempt2 = ctx.getFailedAttempts().get(1);
+ assertTrue(attempt2.getDriverStartTime() - attempt1.getDriverFinishTime() >= 5000);
+ assertTrue(ctx.getDriverStatus().getDriverStartTime() - attempt2.getDriverFinishTime() >= 10000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/query/retry/TestServerRetryPolicyDecider.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/retry/TestServerRetryPolicyDecider.java b/lens-server/src/test/java/org/apache/lens/server/query/retry/TestServerRetryPolicyDecider.java
new file mode 100644
index 0000000..a6e55c9
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/query/retry/TestServerRetryPolicyDecider.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.retry;
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.retry.BackOffRetryHandler;
+import org.apache.lens.server.api.retry.ImmediateRetryHandler;
+import org.apache.lens.server.api.retry.RetryPolicyDecider;
+
+public class TestServerRetryPolicyDecider implements RetryPolicyDecider<QueryContext> {
+
+ @Override
+ public BackOffRetryHandler<QueryContext> decidePolicy(String errorMessage) {
+ return new ImmediateRetryHandler<>(1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/java/org/apache/lens/server/scheduler/util/SchedulerTestUtils.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/scheduler/util/SchedulerTestUtils.java b/lens-server/src/test/java/org/apache/lens/server/scheduler/util/SchedulerTestUtils.java
index d50474a..daea7a2 100644
--- a/lens-server/src/test/java/org/apache/lens/server/scheduler/util/SchedulerTestUtils.java
+++ b/lens-server/src/test/java/org/apache/lens/server/scheduler/util/SchedulerTestUtils.java
@@ -33,8 +33,8 @@ import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.scheduler.*;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryEnded;
import org.apache.lens.server.api.query.QueryExecutionService;
+import org.apache.lens.server.api.query.events.QueryEnded;
import org.apache.lens.server.scheduler.SchedulerServiceImpl;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/resources/drivers/mock/single_failure/failing-query-driver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/mock/single_failure/failing-query-driver-site.xml b/lens-server/src/test/resources/drivers/mock/single_failure/failing-query-driver-site.xml
new file mode 100644
index 0000000..9bb3f1d
--- /dev/null
+++ b/lens-server/src/test/resources/drivers/mock/single_failure/failing-query-driver-site.xml
@@ -0,0 +1,32 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>lens.driver.test.key</name>
+ <value>set</value>
+ </property>
+
+ <property>
+ <name>lens.driver.test.drivername</name>
+ <value>mock/single_failure</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/resources/drivers/retry/double_failure/driver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/retry/double_failure/driver-site.xml b/lens-server/src/test/resources/drivers/retry/double_failure/driver-site.xml
new file mode 100644
index 0000000..c0e6695
--- /dev/null
+++ b/lens-server/src/test/resources/drivers/retry/double_failure/driver-site.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>num.retries</name>
+ <value>2</value>
+ </property>
+ <property>
+ <name>query.retry.policy.classes</name>
+ <value>org.apache.lens.server.query.retry.MockRetryPolicyDecider</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/resources/drivers/retry/single_failure/driver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/retry/single_failure/driver-site.xml b/lens-server/src/test/resources/drivers/retry/single_failure/driver-site.xml
new file mode 100644
index 0000000..e9e0550
--- /dev/null
+++ b/lens-server/src/test/resources/drivers/retry/single_failure/driver-site.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>num.retries</name>
+ <value>1</value>
+ </property>
+ <property>
+ <name>query.retry.policy.classes</name>
+ <value>org.apache.lens.server.query.retry.MockRetryPolicyDecider</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/test/resources/drivers/retry/triple_failure/driver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/retry/triple_failure/driver-site.xml b/lens-server/src/test/resources/drivers/retry/triple_failure/driver-site.xml
new file mode 100644
index 0000000..b7ed287
--- /dev/null
+++ b/lens-server/src/test/resources/drivers/retry/triple_failure/driver-site.xml
@@ -0,0 +1,31 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>num.retries</name>
+ <value>3</value>
+ </property>
+ <property>
+ <name>query.retry.policy.classes</name>
+ <value>org.apache.lens.server.query.retry.MockRetryPolicyDecider</value>
+ </property>
+</configuration>
\ No newline at end of file
[2/4] lens git commit: LENS-743: Query retry framework for retrying
upon transient failures
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
new file mode 100644
index 0000000..6169744
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/StatusChange.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * The Class StatusChange.
+ */
+public abstract class StatusChange extends QueryEvent<QueryStatus.Status> {
+
+ /**
+ * Instantiates a new status change.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public StatusChange(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ }
+
+ /**
+ * Check current state.
+ *
+ * @param status the status
+ */
+ protected void checkCurrentState(QueryStatus.Status status) {
+ if (currentValue != status) {
+ throw new IllegalStateException("Invalid query state: " + currentValue + " query:" + queryHandle.toString());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
new file mode 100644
index 0000000..5ea5710
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/BackOffRetryHandler.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.retry;
+
+import java.io.Serializable;
+
+/**
+ * A backoff retry handler.
+ *
+ * This allows a backoff on any call, so provides methods whether we can try the operation now,
+ * whats next time when operation can be performed and whether operation has exhausted all retries.
+ * Callers of this can do the following :
+ *
+ * if (handler.canTryOpNow(FailureContext)) {
+ * try {
+ * tryCallerOperation();
+ * FailureContext.clear();
+ * } catch (any Transient Exception) {
+ * FailureContext.updateFailure();
+ * if (!handler.hasExhaustedRetries(FailureContext)) {
+ * // will be tried later again
+ * }
+ * throw exception;
+ * }
+ * }
+ *
+ * Note that this is only one of the possible use cases, other complex use cases are in retry framework.
+ */
+public interface BackOffRetryHandler<FC extends FailureContext> extends Serializable {
+
+ /**
+ * To know whether operation can be done now.
+ *
+ * @param failContext FailureContext holding failures till now.
+ *
+ * @return true if operation can be done now, false otherwise.
+ */
+ boolean canTryOpNow(FC failContext);
+
+ /**
+ * Get the time when the operation can be done next.
+ *
+ * @param failContext FC holding failures till now.
+ *
+ * @return Next operation time in millis since epoch
+ */
+ long getOperationNextTime(FC failContext);
+
+ /**
+ * Has the operation exhausted all its retries
+ *
+ * @param failContext FC holding failures till now.
+ *
+ * @return true if all retries have exhausted, false otherwise.
+ */
+ boolean hasExhaustedRetries(FC failContext);
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ChainedRetryPolicyDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ChainedRetryPolicyDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ChainedRetryPolicyDecider.java
new file mode 100644
index 0000000..46526ad
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ChainedRetryPolicyDecider.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.retry;
+
+import java.util.List;
+
+import org.apache.lens.server.api.error.LensException;
+
+import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
+
+import com.google.common.collect.Lists;
+import lombok.Data;
+
+@Data
+public class ChainedRetryPolicyDecider<FC extends FailureContext> implements RetryPolicyDecider<FC> {
+ private final Iterable<RetryPolicyDecider<FC>> policyDeciders;
+
+ @Override
+ public BackOffRetryHandler<FC> decidePolicy(String errorMessage) {
+ for (RetryPolicyDecider<FC> policyDecider : policyDeciders) {
+ BackOffRetryHandler<FC> policy = policyDecider.decidePolicy(errorMessage);
+ if (policy != null) {
+ return policy;
+ }
+ }
+ return new NoRetryHandler<>();
+ }
+ public static <FC extends FailureContext> ChainedRetryPolicyDecider<FC> from(Configuration conf, String key)
+ throws LensException {
+ String[] classNames = conf.getStrings(key);
+ List<RetryPolicyDecider<FC>> retryPolicyDeciders = Lists.newArrayList();
+ if (classNames != null) {
+ for (String className: classNames) {
+ Class<? extends RetryPolicyDecider<FC>> clazz;
+ try {
+ clazz = (Class<? extends RetryPolicyDecider<FC>>) conf.getClassByName(className)
+ .asSubclass(RetryPolicyDecider.class);
+ } catch (ClassNotFoundException e) {
+ throw new LensException("Couldn't load class " + className, e);
+ }
+ RetryPolicyDecider<FC> instance;
+ try {
+ instance = clazz.newInstance();
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Couldn't create instance of class " + clazz.getName(), e);
+ }
+ if (instance instanceof Configurable) {
+ ((Configurable) instance).setConf(conf);
+ }
+ retryPolicyDeciders.add(instance);
+ }
+ }
+ return new ChainedRetryPolicyDecider<>(retryPolicyDeciders);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/DefaultRetryPolicyDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/DefaultRetryPolicyDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/DefaultRetryPolicyDecider.java
new file mode 100644
index 0000000..0f5ba26
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/DefaultRetryPolicyDecider.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.retry;
+
+public class DefaultRetryPolicyDecider<FC extends FailureContext> implements RetryPolicyDecider<FC> {
+
+ @Override
+ public BackOffRetryHandler<FC> decidePolicy(String errorMessage) {
+ return new NoRetryHandler<>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FailureContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FailureContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FailureContext.java
new file mode 100644
index 0000000..24e3d5a
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FailureContext.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.retry;
+
+/**
+ * Any data structure that has fail and retry capability should deal with an implementation of this interface.
+ * @see org.apache.lens.server.api.query.StatusUpdateFailureContext
+ * @see org.apache.lens.server.api.query.QueryContext
+ */
+public interface FailureContext {
+ /**
+ *
+ * @return Last time of failure
+ */
+ long getLastFailedTime();
+
+ /**
+ *
+ * @return number of times failure has occured so far
+ */
+ int getFailCount();
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FibonacciExponentialBackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FibonacciExponentialBackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FibonacciExponentialBackOffRetryHandler.java
new file mode 100644
index 0000000..01da25d
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/FibonacciExponentialBackOffRetryHandler.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.retry;
+
+import static com.google.common.base.Preconditions.checkArgument;
+
+
+/**
+ * A exponential backoff retry handler.
+ *
+ * It allows the the failures to be retried at a next update time, which can increase exponentially.
+ *
+ */
+public class FibonacciExponentialBackOffRetryHandler<FC extends FailureContext> implements BackOffRetryHandler<FC> {
+ final int[] fibonacci;
+ final long maxDelay;
+ final long waitMillis;
+
+ public FibonacciExponentialBackOffRetryHandler(int numRetries, long maxDelay, long waitMillis) {
+ checkArgument(numRetries > 2);
+ fibonacci = new int[numRetries];
+ fibonacci[0] = 1;
+ fibonacci[1] = 1;
+ for(int i = 2; i < numRetries; ++i) {
+ fibonacci[i] = fibonacci[i-1] + fibonacci[i-2];
+ }
+ this.maxDelay = maxDelay;
+ this.waitMillis = waitMillis;
+ }
+
+ public boolean canTryOpNow(FC failContext) {
+ synchronized (failContext) {
+ if (failContext.getFailCount() != 0) {
+ long now = System.currentTimeMillis();
+ if (now < getOperationNextTime(failContext)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+
+ public long getOperationNextTime(FC failContext) {
+ synchronized (failContext) {
+ if (failContext.getFailCount() >= fibonacci.length) {
+ return failContext.getLastFailedTime() + maxDelay;
+ }
+ long delay = Math.min(maxDelay, fibonacci[failContext.getFailCount()] * waitMillis);
+ return failContext.getLastFailedTime() + delay;
+ }
+ }
+
+ public boolean hasExhaustedRetries(FC failContext) {
+ synchronized (failContext) {
+ if (failContext.getFailCount() >= fibonacci.length) {
+ return true;
+ }
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ImmediateRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ImmediateRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ImmediateRetryHandler.java
new file mode 100644
index 0000000..c1c0126
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/ImmediateRetryHandler.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.retry;
+
+import lombok.RequiredArgsConstructor;
+
+@RequiredArgsConstructor
+public class ImmediateRetryHandler<FC extends FailureContext> implements BackOffRetryHandler<FC> {
+ private final int retries;
+ private int retriesDone = 0;
+ // default 1 retry
+ public ImmediateRetryHandler() {
+ this(1);
+ }
+
+ @Override
+ public boolean canTryOpNow(FC failContext) {
+ return true;
+ }
+
+ @Override
+ public long getOperationNextTime(FC failContext) {
+ return System.currentTimeMillis();
+ }
+
+ @Override
+ public boolean hasExhaustedRetries(FC failContext) {
+ return ++retriesDone > retries;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/NoRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/NoRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/NoRetryHandler.java
new file mode 100644
index 0000000..df68a48
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/NoRetryHandler.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.retry;
+
+public class NoRetryHandler<FC extends FailureContext> extends ImmediateRetryHandler<FC> {
+ NoRetryHandler() {
+ super(0);
+ }
+
+ @Override
+ public boolean canTryOpNow(FC failContext) {
+ return false;
+ }
+
+ @Override
+ public long getOperationNextTime(FC failContext) {
+ return Long.MAX_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/OperationRetryHandlerFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/OperationRetryHandlerFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/OperationRetryHandlerFactory.java
new file mode 100644
index 0000000..35fdaca
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/OperationRetryHandlerFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.retry;
+
+/**
+ * Factory which creates operation retry handler
+ */
+public class OperationRetryHandlerFactory {
+ private OperationRetryHandlerFactory() {
+ }
+
+ /**
+ * Create exponential backoff handler
+ *
+ * @param numRetries Number of exponential backoff retries
+ * @param maxDelay Maximum delay an operation can wait for next
+ * @param waitMillis Number of millis that would grow exponentially incase of failures
+ *
+ * @return BackOffRetryHandler
+ */
+ public static <FC extends FailureContext> BackOffRetryHandler<FC> createExponentialBackOffHandler(int numRetries,
+ long maxDelay, long waitMillis) {
+ return new FibonacciExponentialBackOffRetryHandler<>(numRetries, maxDelay, waitMillis);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/retry/RetryPolicyDecider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/retry/RetryPolicyDecider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/RetryPolicyDecider.java
new file mode 100644
index 0000000..88448b0
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/retry/RetryPolicyDecider.java
@@ -0,0 +1,25 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.retry;
+
+
+public interface RetryPolicyDecider<FC extends FailureContext> {
+
+ BackOffRetryHandler<FC> decidePolicy(String errorMessage);
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
index 8261d8a..9d732c1 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/util/LensUtil.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.lens.server.api.common.ConfigBasedObjectCreationFactory;
import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import com.google.common.collect.ImmutableSet;
@@ -69,26 +70,23 @@ public final class LensUtil {
public static boolean isSocketException(@NonNull Throwable e) {
Throwable cause = getCause(e);
- if (cause instanceof SocketException || cause instanceof SocketTimeoutException) {
- return true;
- }
- return false;
+ return cause instanceof SocketException || cause instanceof SocketTimeoutException;
}
- public static <T> ImmutableSet<T> getImplementations(final String factoriesKey, final Configuration conf) {
-
+ public static <T> Set<T> getImplementationsMutable(final String factoriesKey, final Configuration conf) {
Set<T> implSet = Sets.newLinkedHashSet();
final String[] factoryNames = conf.getStrings(factoriesKey);
-
- if (factoryNames == null) {
- return ImmutableSet.copyOf(implSet);
- }
-
- for (String factoryName : factoryNames) {
- if (StringUtils.isNotBlank(factoryName)) {
- final T implementation = getImplementation(factoryName.trim(), conf);
- implSet.add(implementation);
+ if (factoryNames != null) {
+ for (String factoryName : factoryNames) {
+ if (StringUtils.isNotBlank(factoryName)) {
+ final T implementation = getImplementation(factoryName.trim(), conf);
+ implSet.add(implementation);
+ }
}
}
+ return implSet;
+ }
+ public static <T> ImmutableSet<T> getImplementations(final String factoriesKey, final Configuration conf) {
+ Set<T> implSet = getImplementationsMutable(factoriesKey, conf);
return ImmutableSet.copyOf(implSet);
}
@@ -97,7 +95,11 @@ public final class LensUtil {
try {
ConfigBasedObjectCreationFactory<T> factory
= (ConfigBasedObjectCreationFactory<T>) Class.forName(factoryName).newInstance();
- return factory.create(conf);
+ T ret = factory.create(conf);
+ if (ret instanceof Configurable) {
+ ((Configurable) ret).setConf(conf);
+ }
+ return ret;
} catch (final ReflectiveOperationException e) {
throw new IllegalStateException(e);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/test/java/org/apache/lens/server/api/common/TestExponentialBackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/common/TestExponentialBackOffRetryHandler.java b/lens-server-api/src/test/java/org/apache/lens/server/api/common/TestExponentialBackOffRetryHandler.java
deleted file mode 100644
index 5f407af..0000000
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/common/TestExponentialBackOffRetryHandler.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.common;
-
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-
-import org.testng.annotations.Test;
-
-public class TestExponentialBackOffRetryHandler {
-
- @Test
- public void testExponentialBackOff() {
- FailureContext failures = new FailureContext();
- BackOffRetryHandler retryHandler = OperationRetryHandlerFactory.createExponentialBackOffHandler(10, 10000, 1000);
- assertFalse(retryHandler.hasExhaustedRetries(failures));
- assertTrue(retryHandler.canTryOpNow(failures));
-
- long now = System.currentTimeMillis();
- failures.updateFailure();
- assertFalse(retryHandler.hasExhaustedRetries(failures));
- assertFalse(retryHandler.canTryOpNow(failures));
- assertTrue(now + 500 < retryHandler.getOperationNextTime(failures));
- assertTrue(now + 15000 > retryHandler.getOperationNextTime(failures));
-
- for (int i = 0; i < 10; i++) {
- failures.updateFailure();
- }
- assertTrue(retryHandler.hasExhaustedRetries(failures));
- assertFalse(retryHandler.canTryOpNow(failures));
-
- failures.clear();
- assertFalse(retryHandler.hasExhaustedRetries(failures));
- assertTrue(retryHandler.canTryOpNow(failures));
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 3c18ac7..2a2963f 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -58,7 +58,7 @@ public class MockDriver extends AbstractLensDriver {
/**
* The conf.
*/
- Configuration conf;
+ protected Configuration conf;
/**
* The query.
@@ -99,7 +99,7 @@ public class MockDriver extends AbstractLensDriver {
this.conf = conf;
ioTestVal = conf.getInt("mock.driver.test.val", -1);
this.conf.addResource(getDriverResourcePath("failing-query-driver-site.xml"));
- getQueryHook().setDriver(this);
+ loadQueryHook();
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
index a530e9d..1560bf1 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/TestQueryContext.java
@@ -24,10 +24,11 @@ import static org.testng.Assert.*;
import java.util.List;
import org.apache.lens.api.LensConf;
-import org.apache.lens.server.api.common.*;
import org.apache.lens.server.api.driver.LensDriver;
import org.apache.lens.server.api.driver.MockDriver;
import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.retry.BackOffRetryHandler;
+import org.apache.lens.server.api.retry.FibonacciExponentialBackOffRetryHandler;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/test/java/org/apache/lens/server/api/query/comparators/ChainedComparatorTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/comparators/ChainedComparatorTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/comparators/ChainedComparatorTest.java
new file mode 100644
index 0000000..cc58751
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/comparators/ChainedComparatorTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.comparators;
+
+import static org.testng.Assert.*;
+
+import java.util.Comparator;
+
+import org.testng.annotations.DataProvider;
+import org.testng.annotations.Test;
+
+import com.google.common.collect.Lists;
+import lombok.Data;
+
+public class ChainedComparatorTest {
+ @Data
+ private static class Tuple {
+ final Integer a, b, c;
+ }
+
+ private Tuple tuple(Integer a, Integer b, Integer c) {
+ return new Tuple(a, b, c);
+ }
+
+ public static final ChainedComparator<Tuple> COMPARATOR = new ChainedComparator<>(Lists.newArrayList(
+ new Comparator<Tuple>() {
+ @Override
+ public int compare(Tuple o1, Tuple o2) {
+ return o1.getA().compareTo(o2.getA());
+ }
+ },
+ new Comparator<Tuple>() {
+
+ @Override
+ public int compare(Tuple o1, Tuple o2) {
+ return o1.getB().compareTo(o2.getB());
+ }
+ },
+ new Comparator<Tuple>() {
+ @Override
+ public int compare(Tuple o1, Tuple o2) {
+ return o1.getC().compareTo(o2.getC());
+ }
+ }
+ ));
+
+ @DataProvider
+ public Object[][] comparisonData() {
+ return new Object[][]{
+ {tuple(0, 0, 0), tuple(0, 0, 0), 0},
+ {tuple(0, 0, 1), tuple(0, 0, 0), 1},
+ {tuple(0, 0, 1), tuple(0, 0, 4), -1},
+ {tuple(0, 0, 1), tuple(1, 0, 4), -1},
+ {tuple(0, 0, 1), tuple(0, -10, 4), 1},
+ };
+ }
+
+ @Test(dataProvider = "comparisonData")
+ public void testCompare(Tuple a, Tuple b, int expected) throws Exception {
+ assertEquals(COMPARATOR.compare(a, b), expected);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
index 122409b..2667ebf 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraintTest.java
@@ -24,7 +24,8 @@ import static org.apache.lens.server.api.LensServerAPITestUtil.getConfiguration;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertNotNull;
+import static org.testng.Assert.assertNull;
import java.util.HashSet;
import java.util.Set;
@@ -164,9 +165,13 @@ public class MaxConcurrentDriverQueriesConstraintTest {
when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver);
when(mockLaunchedQueries.getQueriesCount(mockDriver)).thenReturn(currentDriverLaunchedQueries);
- boolean actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
+ String actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
- assertEquals(actualCanLaunch, expectedCanLaunch);
+ if (expectedCanLaunch) {
+ assertNull(actualCanLaunch);
+ } else {
+ assertNotNull(actualCanLaunch);
+ }
}
@Test(dataProvider = "dpTestConcurrentLaunches")
@@ -186,9 +191,13 @@ public class MaxConcurrentDriverQueriesConstraintTest {
when(mockLaunchedQueries.getQueriesCount(mockDriver)).thenReturn(currentDriverLaunchedQueries);
when(mockLaunchedQueries.getQueries(mockDriver)).thenReturn(queries);
- boolean actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
+ String actualCanLaunch = constraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
- assertEquals(actualCanLaunch, expectedCanLaunch);
+ if (expectedCanLaunch) {
+ assertNull(actualCanLaunch);
+ } else {
+ assertNotNull(actualCanLaunch);
+ }
}
@Test(dataProvider = "dpTestPerQueueConstraints")
@@ -208,9 +217,13 @@ public class MaxConcurrentDriverQueriesConstraintTest {
QueryContext mockCandidateQuery = mock(QueryContext.class);
when(mockCandidateQuery.getQueue()).thenReturn(candidateQueue);
when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver);
- boolean actualCanLaunch = perQueueConstraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
+ String actualCanLaunch = perQueueConstraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
- assertEquals(actualCanLaunch, expectedCanLaunch);
+ if (expectedCanLaunch) {
+ assertNull(actualCanLaunch);
+ } else {
+ assertNotNull(actualCanLaunch);
+ }
}
@Test(dataProvider = "dpTestPerPriorityConstraints")
@@ -230,9 +243,13 @@ public class MaxConcurrentDriverQueriesConstraintTest {
QueryContext mockCandidateQuery = mock(QueryContext.class);
when(mockCandidateQuery.getPriority()).thenReturn(candidatePriority);
when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver);
- boolean actualCanLaunch = perPriorityConstraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
+ String actualCanLaunch = perPriorityConstraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
- assertEquals(actualCanLaunch, expectedCanLaunch);
+ if (expectedCanLaunch) {
+ assertNull(actualCanLaunch);
+ } else {
+ assertNotNull(actualCanLaunch);
+ }
}
@Test(dataProvider = "dpTestPerQueuePerPriorityConstraints")
@@ -254,8 +271,12 @@ public class MaxConcurrentDriverQueriesConstraintTest {
when(mockCandidateQuery.getQueue()).thenReturn(candidateQueue);
when(mockCandidateQuery.getPriority()).thenReturn(candidatePriority);
when(mockCandidateQuery.getSelectedDriver()).thenReturn(mockDriver);
- boolean actualCanLaunch = perQueueAndPerPriorityConstraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
+ String actualCanLaunch = perQueueAndPerPriorityConstraint.allowsLaunchOf(mockCandidateQuery, mockLaunchedQueries);
- assertEquals(actualCanLaunch, expectedCanLaunch);
+ if (expectedCanLaunch) {
+ assertNull(actualCanLaunch);
+ } else {
+ assertNotNull(actualCanLaunch);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/test/java/org/apache/lens/server/api/retry/TestExponentialBackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/retry/TestExponentialBackOffRetryHandler.java b/lens-server-api/src/test/java/org/apache/lens/server/api/retry/TestExponentialBackOffRetryHandler.java
new file mode 100644
index 0000000..26261dd
--- /dev/null
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/retry/TestExponentialBackOffRetryHandler.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.retry;
+
+import static org.testng.Assert.assertFalse;
+import static org.testng.Assert.assertTrue;
+
+import org.apache.lens.server.api.query.StatusUpdateFailureContext;
+
+import org.testng.annotations.Test;
+
+public class TestExponentialBackOffRetryHandler {
+
+ @Test
+ public void testExponentialBackOff() {
+ StatusUpdateFailureContext failures = new StatusUpdateFailureContext();
+ BackOffRetryHandler<StatusUpdateFailureContext> retryHandler
+ = OperationRetryHandlerFactory.createExponentialBackOffHandler(10, 10000, 1000);
+ assertFalse(retryHandler.hasExhaustedRetries(failures));
+ assertTrue(retryHandler.canTryOpNow(failures));
+
+ long now = System.currentTimeMillis();
+ failures.updateFailure();
+ assertFalse(retryHandler.hasExhaustedRetries(failures));
+ assertFalse(retryHandler.canTryOpNow(failures));
+ assertTrue(now + 500 < retryHandler.getOperationNextTime(failures));
+ assertTrue(now + 15000 > retryHandler.getOperationNextTime(failures));
+
+ for (int i = 0; i < 10; i++) {
+ failures.updateFailure();
+ }
+ assertTrue(retryHandler.hasExhaustedRetries(failures));
+ assertFalse(retryHandler.canTryOpNow(failures));
+
+ failures.clear();
+ assertFalse(retryHandler.hasExhaustedRetries(failures));
+ assertTrue(retryHandler.canTryOpNow(failures));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/pom.xml
----------------------------------------------------------------------
diff --git a/lens-server/pom.xml b/lens-server/pom.xml
index 6dea9a7..d24dc1e 100644
--- a/lens-server/pom.xml
+++ b/lens-server/pom.xml
@@ -372,7 +372,7 @@
<environmentVariables>
<MVN_CLASSPATH_FILE>${mvn.classpath.file}</MVN_CLASSPATH_FILE>
</environmentVariables>
- <argLine>-Xms256m -Xmx512m -XX:PermSize=256m -XX:MaxPermSize=256m</argLine>
+ <argLine>-Xms256m -Xmx1024m -XX:PermSize=256m -XX:MaxPermSize=256m</argLine>
</configuration>
</plugin>
<plugin>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
index b88c717..9f14396 100644
--- a/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/metrics/MetricsServiceImpl.java
@@ -39,7 +39,7 @@ import org.apache.lens.server.api.health.HealthStatus;
import org.apache.lens.server.api.metastore.CubeMetastoreService;
import org.apache.lens.server.api.metrics.*;
import org.apache.lens.server.api.query.QueryExecutionService;
-import org.apache.lens.server.api.query.StatusChange;
+import org.apache.lens.server.api.query.events.StatusChange;
import org.apache.lens.server.api.session.*;
import org.apache.lens.server.healthcheck.LensServiceHealthCheck;
import org.apache.lens.server.query.QueryExecutionServiceImpl;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java b/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java
deleted file mode 100644
index 75c1146..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/query/FIFOQueryComparator.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.query;
-
-import org.apache.lens.server.api.query.QueryContext;
-
-public class FIFOQueryComparator implements QueryComparator {
-
- @Override
- public int compare(QueryContext o1, QueryContext o2) {
-
- Long submitTimeO1 = o1.getSubmissionTime();
- Long submitTimeO2 = o2.getSubmissionTime();
-
- return submitTimeO1.compareTo(submitTimeO2);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index a540c3c..3ab3aef 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -28,6 +28,7 @@ import java.util.List;
import javax.sql.DataSource;
import org.apache.lens.api.LensConf;
+import org.apache.lens.api.query.FailedAttempt;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.error.LensException;
@@ -35,15 +36,12 @@ import org.apache.lens.server.api.query.FinishedLensQuery;
import org.apache.lens.server.util.UtilityMethods;
import org.apache.commons.codec.binary.Base64;
-import org.apache.commons.dbutils.BasicRowProcessor;
-import org.apache.commons.dbutils.BeanProcessor;
-import org.apache.commons.dbutils.QueryRunner;
-import org.apache.commons.dbutils.ResultSetHandler;
-import org.apache.commons.dbutils.RowProcessor;
+import org.apache.commons.dbutils.*;
import org.apache.commons.dbutils.handlers.BeanHandler;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import com.google.common.collect.Lists;
import lombok.extern.slf4j.Slf4j;
/**
@@ -93,7 +91,7 @@ public class LensServerDAO {
+ "metadata varchar(100000), " + "rows int, " + "filesize bigint, " + "errormessage varchar(10000), "
+ "driverstarttime bigint, " + "driverendtime bigint, " + "drivername varchar(10000), "
+ "queryname varchar(255), " + "submissiontime bigint, " + "driverquery varchar(1000000), "
- + "conf varchar(100000))";
+ + "conf varchar(100000), numfailedattempts int)";
try {
QueryRunner runner = new QueryRunner(ds);
runner.update(sql);
@@ -102,6 +100,18 @@ public class LensServerDAO {
log.warn("Unable to create finished queries table", e);
}
}
+ public void createFailedAttemptsTable() throws Exception {
+ String sql = "CREATE TABLE if not exists failed_attempts (handle varchar(255) not null,"
+ + "attempt_number int, drivername varchar(10000), progress float, progressmessage varchar(10000), "
+ + "errormessage varchar(10000), driverstarttime bigint, driverendtime bigint)";
+ try {
+ QueryRunner runner = new QueryRunner(ds);
+ runner.update(sql);
+ log.info("Created failed_attempts table");
+ } catch (SQLException e) {
+ log.error("Unable to create failed_attempts table", e);
+ }
+ }
/**
* DAO method to insert a new Finished query into Table.
@@ -115,14 +125,26 @@ public class LensServerDAO {
// The expected case
String sql = "insert into finished_queries (handle, userquery, submitter, priority, "
+ "starttime,endtime,result,status,metadata,rows,filesize,"
- + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime, driverquery, conf)"
- + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
- QueryRunner runner = new QueryRunner(ds);
- runner.update(sql, query.getHandle(), query.getUserQuery(), query.getSubmitter(), query.getPriority(),
- query.getStartTime(), query.getEndTime(), query.getResult(), query.getStatus(), query.getMetadata(),
- query.getRows(), query.getFileSize(), query.getErrorMessage(), query.getDriverStartTime(),
- query.getDriverEndTime(), query.getDriverName(), query.getQueryName(), query.getSubmissionTime(),
- query.getDriverQuery(), serializeConf(query.getConf()));
+ + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime, driverquery, conf, "
+ + "numfailedattempts)"
+ + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+ Connection conn = null;
+ try {
+ conn = getConnection();
+ conn.setAutoCommit(false);
+ QueryRunner runner = new QueryRunner();
+ runner.update(conn, sql, query.getHandle(), query.getUserQuery(), query.getSubmitter(), query.getPriority(),
+ query.getStartTime(), query.getEndTime(), query.getResult(), query.getStatus(), query.getMetadata(),
+ query.getRows(), query.getFileSize(), query.getErrorMessage(), query.getDriverStartTime(),
+ query.getDriverEndTime(), query.getDriverName(), query.getQueryName(), query.getSubmissionTime(),
+ query.getDriverQuery(), serializeConf(query.getConf()), query.getFailedAttempts().size());
+ for (int i = 0; i < query.getFailedAttempts().size(); i++) {
+ insertFailedAttempt(runner, conn, query.getHandle(), query.getFailedAttempts().get(i), i);
+ }
+ conn.commit();
+ } finally {
+ DbUtils.closeQuietly(conn);
+ }
} else {
log.warn("Re insert happening in purge: " + Thread.currentThread().getStackTrace());
if (alreadyExisting.equals(query)) {
@@ -135,6 +157,50 @@ public class LensServerDAO {
}
}
}
+ /**
+ * DAO method to insert a new Finished query into Table.
+ *
+ *
+ * @param runner
+ * @param conn
+ *@param handle to be inserted
+ * @param index @throws SQLException the exception
+ */
+ public void insertFailedAttempt(QueryRunner runner, Connection conn, String handle, FailedAttempt attempt, int index)
+ throws SQLException {
+ String sql = "insert into failed_attempts(handle, attempt_number, drivername, progress, progressmessage, "
+ + "errormessage, driverstarttime, driverendtime) values (?, ?, ?, ?, ?, ?, ?, ?)";
+ runner.update(conn, sql, handle, index, attempt.getDriverName(),
+ attempt.getProgress(), attempt.getProgressMessage(), attempt.getErrorMessage(),
+ attempt.getDriverStartTime(), attempt.getDriverFinishTime());
+ }
+
+ public void getFailedAttempts(final FinishedLensQuery query) {
+ if (query != null) {
+ String handle = query.getHandle();
+ ResultSetHandler<List<FailedAttempt>> rsh = new BeanHandler<List<FailedAttempt>>(null) {
+ @Override
+ public List<FailedAttempt> handle(ResultSet rs) throws SQLException {
+ List<FailedAttempt> attempts = Lists.newArrayList();
+ while (rs.next()) {
+ FailedAttempt attempt = new FailedAttempt(rs.getString(3), rs.getDouble(4), rs.getString(5),
+ rs.getString(6), rs.getLong(7), rs.getLong(8));
+ attempts.add(attempt);
+ }
+ return attempts;
+ }
+ };
+ String sql = "select * from failed_attempts where handle=? order by attempt_number";
+ QueryRunner runner = new QueryRunner(ds);
+ try {
+ query.setFailedAttempts(runner.query(sql, rsh, handle));
+ } catch (SQLException e) {
+ log.error("SQL exception while executing query.", e);
+ }
+ }
+ }
+
+
private String serializeConf(LensConf conf) {
return Base64.encodeBase64String(conf.toXMLString().getBytes(Charset.defaultCharset()));
@@ -157,7 +223,9 @@ public class LensServerDAO {
String sql = "select * from finished_queries where handle=?";
QueryRunner runner = new QueryRunner(ds);
try {
- return runner.query(sql, rsh, handle);
+ FinishedLensQuery finishedQuery = runner.query(sql, rsh, handle);
+ getFailedAttempts(finishedQuery);
+ return finishedQuery;
} catch (SQLException e) {
log.error("SQL exception while executing query.", e);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java
deleted file mode 100644
index 67dda6b..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryComparator.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-
-package org.apache.lens.server.query;
-
-import java.util.Comparator;
-
-import org.apache.lens.server.api.query.QueryContext;
-
-public interface QueryComparator extends Comparator<QueryContext> {
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryCostComparator.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryCostComparator.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryCostComparator.java
deleted file mode 100644
index 2702581..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryCostComparator.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.query;
-
-import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.cost.QueryCost;
-
-public class QueryCostComparator extends FIFOQueryComparator {
-
- @Override
- public int compare(final QueryContext o1, final QueryContext o2) {
-
- QueryCost qcO1 = o1.getSelectedDriverQueryCost();
- QueryCost qcO2 = o2.getSelectedDriverQueryCost();
-
- int result = qcO1.compareTo(qcO2);
- if (result == 0) {
- return super.compare(o1, o2);
- }
- return result;
- }
-}
-
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java
index 5d2ddbe..e932672 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndHttpNotifier.java
@@ -24,8 +24,8 @@ import java.util.Map;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryEnded;
-import org.apache.lens.server.api.query.QueryEvent;
+import org.apache.lens.server.api.query.events.QueryEnded;
+import org.apache.lens.server.api.query.events.QueryEvent;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
index 91fddc9..2a34c68 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEndNotifier.java
@@ -40,7 +40,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryEnded;
+import org.apache.lens.server.api.query.events.QueryEnded;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.commons.lang3.StringUtils;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
index f264603..1760bec 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryEventHttpNotifier.java
@@ -36,8 +36,8 @@ import org.apache.lens.api.util.MoxyJsonConfigurationContextResolver;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryEnded;
-import org.apache.lens.server.api.query.QueryEvent;
+import org.apache.lens.server.api.query.events.QueryEnded;
+import org.apache.lens.server.api.query.events.QueryEvent;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index cb5961f..b5e996f 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -50,8 +50,6 @@ import org.apache.lens.server.BaseLensService;
import org.apache.lens.server.LensServerConf;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.common.BackOffRetryHandler;
-import org.apache.lens.server.api.common.OperationRetryHandlerFactory;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.error.LensMultiCauseException;
@@ -62,8 +60,11 @@ import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.comparators.*;
import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.query.events.*;
+import org.apache.lens.server.api.retry.*;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.lens.server.model.MappedDiagnosticLogSegregationContext;
@@ -235,7 +236,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
/**
* The query comparator
*/
- private QueryComparator queryComparator;
+ private Comparator<QueryContext> queryComparator;
/**
* The result sets.
*/
@@ -295,7 +296,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
* */
private final ReentrantLock removalFromLaunchedQueriesLock = new ReentrantLock();
- private final ExecutorService waitingQueriesSelectionSvc = Executors.newSingleThreadExecutor();
+ private final ScheduledExecutorService waitingQueriesSelectionSvc = Executors.newSingleThreadScheduledExecutor();
/**
* This is the TTL millis for all result sets of type {@link org.apache.lens.server.api.driver.InMemoryResultSet}
@@ -321,7 +322,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
private UserQueryToCubeQueryRewriter userQueryToCubeQueryRewriter;
// Exponential backoff retry handler for status updates
- private BackOffRetryHandler statusUpdateRetryHandler;
+ private BackOffRetryHandler<StatusUpdateFailureContext> statusUpdateRetryHandler;
+ private RetryPolicyDecider<QueryContext> queryRetryPolicyDecider;
/**
* Instantiates a new query execution service impl.
@@ -376,9 +378,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
*
* @throws LensException the lens exception
*/
- private void loadDriversAndSelector() throws LensException {
- //Load all configured Drivers
- loadDrivers();
+ private void loadDriverSelector() throws LensException {
//Load configured Driver Selector
try {
Class<? extends DriverSelector> driverSelectorClass = conf.getClass(DRIVER_SELECTOR_CLASS,
@@ -394,14 +394,18 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
}
private void loadQueryComparator() throws LensException {
try {
- Class<? extends QueryComparator> queryComparatorClass = conf.getClass(QUERY_COMPARATOR_CLASS,
- QueryPriorityComparator.class, QueryComparator.class);
- log.info("Using query comparator class: {}", queryComparatorClass.getCanonicalName());
- queryComparator = queryComparatorClass.newInstance();
+ Class<?>[] classes = conf.getClasses(QUERY_COMPARATOR_CLASSES,
+ MoreRetriesFirstComparator.class, QueryPriorityComparator.class,
+ FIFOQueryComparator.class, QueryCostComparator.class);
+ List<Comparator<QueryContext>> comparators = Lists.newArrayList();
+ for (Class<?> clazz: classes) {
+ comparators.add(clazz.asSubclass(QueryComparator.class).newInstance());
+ }
+ queryComparator = new ChainedComparator<>(comparators);
} catch (Exception e) {
- throw new LensException("Couldn't instantiate query comparator class. Class name: "
- + conf.get(QUERY_COMPARATOR_CLASS) + ". Please supply a valid value for "
- + QUERY_COMPARATOR_CLASS);
+ throw new LensException("Couldn't instantiate query comparator class. Classes: "
+ + conf.get(QUERY_COMPARATOR_CLASSES) + ". Please supply a valid value for "
+ + QUERY_COMPARATOR_CLASSES);
}
}
@@ -681,9 +685,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
Thread.sleep(100);
continue;
}
- QueryContext query = queuedQueries.take();
+ final QueryContext query = queuedQueries.take();
synchronized (query) {
-
/* Setting log segregation id */
logSegregationContext.setLogSegragationAndQueryId(query.getQueryHandleString());
@@ -714,6 +717,17 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
launched queries. First add to waiting queries, then release lock */
addToWaitingQueries(query);
removalFromLaunchedQueriesLock.unlock();
+ if (query.getRetryPolicy() != null) {
+ waitingQueriesSelectionSvc.schedule(new Runnable() {
+ @Override
+ public void run() {
+ if (waitingQueries.remove(query)) {
+ queuedQueries.add(query);
+ }
+ }
+ }, query.getRetryPolicy().getOperationNextTime(query) - System.currentTimeMillis(),
+ TimeUnit.MILLISECONDS);
+ }
}
} finally {
if (removalFromLaunchedQueriesLock.isHeldByCurrentThread()) {
@@ -881,6 +895,43 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
log.info("StatusPoller exited");
}
}
+ private boolean handleRetries(QueryContext ctx) throws LensException {
+ // TODO: handle retries for post-processing, e.g. result formatting failure doesn't need query rerun
+ if (ctx.getStatus().failing()) {
+ if (removeFromLaunchedQueries(ctx)) {
+ processWaitingQueriesAsync(ctx);
+ }
+ if (ctx.getDriverStatus().failed() && !getDriverRetryPolicy(ctx).hasExhaustedRetries(ctx)) {
+ log.info("query {} will be retried on the same driver {}",
+ ctx.getQueryHandle(), ctx.getSelectedDriver().getFullyQualifiedName());
+ ctx.extractFailedAttempt();
+ ctx.setStatus(QueryStatus.getQueuedStatus());
+ ctx.getSelectedDriver().closeQuery(ctx.getQueryHandle());
+ return queuedQueries.add(ctx);
+ } else if (!getServerRetryPolicy(ctx).hasExhaustedRetries(ctx)) {
+ LensDriver selectedDriver = ctx.getSelectedDriver();
+ ctx.getDriverContext().blacklist(selectedDriver);
+ try (SessionContext ignored = new SessionContext(getSessionHandle(ctx.getLensSessionIdentifier()))) {
+ rewriteAndSelect(ctx);
+ } catch (LensException e) {
+ log.error("driver {} gave up on query {} and it will not be retried on any other driver since rewrite failed",
+ selectedDriver.getFullyQualifiedName(), e);
+ ctx.setStatus(new QueryStatus(1.0f, null, FAILED, ctx.getStatus().getStatusMessage(), false, null,
+ ctx.getStatus().getErrorMessage(), ctx.getStatus().getLensErrorTO()));
+ return false;
+ }
+ log.info("driver {} gave up on query {} and it will be retried on {}", selectedDriver.getFullyQualifiedName(),
+ ctx.getQueryHandle(), ctx.getSelectedDriver().getFullyQualifiedName());
+ ctx.extractFailedAttempt(selectedDriver);
+ ctx.setStatus(QueryStatus.getQueuedStatus());
+ selectedDriver.closeQuery(ctx.getQueryHandle());
+ return queuedQueries.add(ctx);
+ }
+ ctx.setStatus(new QueryStatus(1.0f, null, FAILED, ctx.getStatus().getStatusMessage(), false, null,
+ ctx.getStatus().getErrorMessage(), ctx.getStatus().getLensErrorTO()));
+ }
+ return false;
+ }
/**
* Sets the failed status.
@@ -891,13 +942,33 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
* @throws LensException the lens exception
*/
void setFailedStatus(QueryContext ctx, String statusMsg, Exception e) throws LensException {
-
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0f, null, FAILED, statusMsg, false, null, LensUtil.getCauseMessage(e),
+ ctx.setStatus(new QueryStatus(0.0f, null, FAILING, statusMsg, false, null, LensUtil.getCauseMessage(e),
e instanceof LensException ? ((LensException)e).buildLensErrorTO(this.errorCollection) : null));
- updateFinishedQuery(ctx, before);
+ handleRetries(ctx);
+ if (ctx.finished()) {
+ updateFinishedQuery(ctx, before);
+ }
fireStatusChangeEvent(ctx, ctx.getStatus(), before);
}
+
+ private BackOffRetryHandler<QueryContext> getServerRetryPolicy(QueryContext ctx) {
+ if (ctx.getServerRetryPolicy() == null) {
+ // allow new driver to retry
+ ctx.setDriverRetryPolicy(null);
+ ctx.setServerRetryPolicy(queryRetryPolicyDecider.decidePolicy(ctx.getStatus().getErrorMessage()));
+ }
+ return ctx.getServerRetryPolicy();
+ }
+
+ private BackOffRetryHandler<QueryContext> getDriverRetryPolicy(QueryContext ctx) {
+ if (ctx.getDriverRetryPolicy() == null) {
+ ctx.setDriverRetryPolicy(ctx.getSelectedDriver().getRetryPolicyDecider()
+ .decidePolicy(ctx.getDriverStatus().getErrorMessage()));
+ }
+ return ctx.getDriverRetryPolicy();
+ }
+
/**
* Sets the cancelled status.
*
@@ -984,6 +1055,9 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
|| !ctx.isResultAvailableInDriver())) {
setSuccessState(ctx);
} else {
+ if (ctx.getStatus().failing()) {
+ handleRetries(ctx);
+ }
if (ctx.getStatus().finished()) {
updateFinishedQuery(ctx, before);
}
@@ -1029,7 +1103,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
case LAUNCHED:
return new QueryLaunched(ctx.getLaunchTime(), prevState, currState, query);
case QUEUED:
- return new QueryQueued(ctx.getSubmissionTime(), prevState, currState, query, ctx.getSubmittedUser());
+ if (ctx.getFailedAttempts().size() > 0) {
+ return new QueryQueuedForRetry(ctx.getSubmissionTime(), prevState, currState, query, ctx.getSubmittedUser());
+ } else {
+ return new QueryQueued(ctx.getSubmissionTime(), prevState, currState, query, ctx.getSubmittedUser());
+ }
case RUNNING:
return new QueryRunning(System.currentTimeMillis() - ctx.getDriverStatus().getDriverStartTime(), prevState,
currState, query);
@@ -1239,13 +1317,21 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
throw new IllegalStateException("Could not load phase 1 rewriters");
}
try {
+ loadQueryRetryPolicyDecider(conf);
+ } catch (LensException e) {
+ throw new IllegalStateException("Could not load retry policy", e);
+ }
+ try {
initializeQueryAcceptors();
} catch (LensException e) {
throw new IllegalStateException("Could not load acceptors");
}
initializeListeners();
try {
- loadDriversAndSelector();
+ // Load all configured Drivers
+ loadDrivers();
+ // load driver selector
+ loadDriverSelector();
} catch (LensException e) {
log.error("Error while loading drivers", e);
throw new IllegalStateException("Could not load drivers", e);
@@ -1269,6 +1355,10 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
log.info("Query execution service initialized");
}
+ private void loadQueryRetryPolicyDecider(Configuration conf) throws LensException {
+ this.queryRetryPolicyDecider = ChainedRetryPolicyDecider.from(conf, QUERY_RETRY_POLICY_CLASSES);
+ }
+
/**
* Initalize finished query store.
*
@@ -1279,8 +1369,9 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
this.lensServerDao.init(conf);
try {
this.lensServerDao.createFinishedQueriesTable();
+ this.lensServerDao.createFailedAttemptsTable();
} catch (Exception e) {
- log.warn("Unable to create finished query table, query purger will not purge queries", e);
+ log.warn("Unable to create finished query tables, query purger will not purge queries", e);
}
}
@@ -1429,15 +1520,16 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
log.info("Recovered {} queries", allQueries.size());
}
super.start();
- querySubmitter.start();
- statusPoller.start();
- queryPurger.start();
- prepareQueryPurger.start();
startEstimatePool();
startLauncherPool();
startQueryCancellationPool();
+ querySubmitter.start();
+ statusPoller.start();
+ queryPurger.start();
+ prepareQueryPurger.start();
+
if (conf.getBoolean(RESULTSET_PURGE_ENABLED, DEFAULT_RESULTSET_PURGE_ENABLED)) {
queryResultPurger = new QueryResultPurger();
queryResultPurger.init(conf);
@@ -1598,8 +1690,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
// Evaluate success of rewrite and estimate
boolean succeededOnce = false;
- List<String> failureCauses = new ArrayList<String>(numDrivers);
- List<LensException> causes = new ArrayList<LensException>(numDrivers);
+ List<String> failureCauses = new ArrayList<>(numDrivers);
+ List<LensException> causes = new ArrayList<>(numDrivers);
for (RewriteEstimateRunnable r : runnables) {
if (r.isSucceeded()) {
@@ -2087,7 +2179,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
private QueryHandle submitQuery(final QueryContext ctx) throws LensException {
synchronized (ctx) {
QueryStatus before = ctx.getStatus();
- ctx.setStatus(new QueryStatus(0.0, null, QUEUED, "Query is queued", false, null, null, null));
+ ctx.setStatus(QueryStatus.getQueuedStatus());
queuedQueries.add(ctx);
log.info("Added to Queued Queries:{}", ctx.getQueryHandleString());
allQueries.put(ctx.getQueryHandle(), ctx);
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionStatisticsGenerator.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionStatisticsGenerator.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionStatisticsGenerator.java
index 55cabe2..557daa2 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionStatisticsGenerator.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionStatisticsGenerator.java
@@ -24,7 +24,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.events.LensEventService;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryEnded;
+import org.apache.lens.server.api.query.events.QueryEnded;
import org.apache.lens.server.stats.event.query.QueryDriverStatistics;
import org.apache.lens.server.stats.event.query.QueryExecutionStatistics;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/QueryPriorityComparator.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryPriorityComparator.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryPriorityComparator.java
deleted file mode 100644
index 2c6d904..0000000
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryPriorityComparator.java
+++ /dev/null
@@ -1,38 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.query;
-
-import org.apache.lens.api.Priority;
-import org.apache.lens.server.api.query.QueryContext;
-
-public class QueryPriorityComparator extends FIFOQueryComparator {
-
- @Override
- public int compare(final QueryContext o1, final QueryContext o2) {
-
- Priority pO1 = o1.getPriority();
- Priority pO2 = o2.getPriority();
-
- int result = pO1.compareTo(pO2);
- if (result == 0) {
- return super.compare(o1, o2);
- }
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
index 41cf33b..c7dc0e1 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/ResultFormatter.java
@@ -28,6 +28,7 @@ import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.metrics.MetricsService;
import org.apache.lens.server.api.query.*;
+import org.apache.lens.server.api.query.events.QueryExecuted;
import org.apache.lens.server.model.LogSegregationContext;
import org.apache.hadoop.fs.FileSystem;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java b/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java
index 48291b9..cf117dc 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/constraint/DefaultQueryLaunchingConstraintsChecker.java
@@ -19,11 +19,13 @@
package org.apache.lens.server.query.constraint;
+import java.util.Collections;
import java.util.Set;
import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.retry.BackOffRetryHandler;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableSet;
@@ -44,7 +46,7 @@ public class DefaultQueryLaunchingConstraintsChecker implements QueryLaunchingCo
private final ImmutableSet<QueryLaunchingConstraint> lensQueryConstraints;
public DefaultQueryLaunchingConstraintsChecker(
- @NonNull final ImmutableSet<QueryLaunchingConstraint> lensQueryConstraints) {
+ @NonNull final ImmutableSet<QueryLaunchingConstraint> lensQueryConstraints) {
this.lensQueryConstraints = lensQueryConstraints;
}
@@ -54,8 +56,11 @@ public class DefaultQueryLaunchingConstraintsChecker implements QueryLaunchingCo
Set<QueryLaunchingConstraint> allConstraints = prepareAllConstraints(candidateQuery);
for (QueryLaunchingConstraint queryConstraint : allConstraints) {
- if (!queryConstraint.allowsLaunchOf(candidateQuery, launchedQueries)) {
- log.info("query {} not allowed to launch. Constraint failed: {}", candidateQuery, queryConstraint);
+ String launchRejectionMessage = queryConstraint.allowsLaunchOf(candidateQuery, launchedQueries);
+ if (launchRejectionMessage != null) {
+ log.info("query {} not allowed to launch. Constraint failed: {} with message: {}",
+ candidateQuery, queryConstraint, launchRejectionMessage);
+ candidateQuery.getStatus().setProgressMessage(launchRejectionMessage);
return false;
}
}
@@ -66,6 +71,12 @@ public class DefaultQueryLaunchingConstraintsChecker implements QueryLaunchingCo
Set<QueryLaunchingConstraint> prepareAllConstraints(final QueryContext candidateQuery) {
ImmutableSet<QueryLaunchingConstraint> driverConstraints = candidateQuery.getSelectedDriverQueryConstraints();
- return Sets.union(this.lensQueryConstraints, driverConstraints);
+ BackOffRetryHandler<QueryContext> retryPolicy = candidateQuery.getRetryPolicy();
+ Sets.SetView<QueryLaunchingConstraint> constraints = Sets.union(this.lensQueryConstraints, driverConstraints);
+ if (retryPolicy == null) {
+ return constraints;
+ } else {
+ return Sets.union(Collections.singleton(new RetryPolicyToConstraingAdapter(retryPolicy)), constraints);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/constraint/RetryPolicyToConstraingAdapter.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/constraint/RetryPolicyToConstraingAdapter.java b/lens-server/src/main/java/org/apache/lens/server/query/constraint/RetryPolicyToConstraingAdapter.java
new file mode 100644
index 0000000..e0d6d80
--- /dev/null
+++ b/lens-server/src/main/java/org/apache/lens/server/query/constraint/RetryPolicyToConstraingAdapter.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.query.constraint;
+
+
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.EstimatedImmutableQueryCollection;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.retry.BackOffRetryHandler;
+
+import lombok.Data;
+
+@Data
+public class RetryPolicyToConstraingAdapter implements QueryLaunchingConstraint {
+ private final BackOffRetryHandler<QueryContext> constraint;
+ @Override
+ public String allowsLaunchOf(QueryContext candidateQuery, EstimatedImmutableQueryCollection launchedQueries) {
+ if (!constraint.canTryOpNow(candidateQuery)) {
+ return "Query will be automatically re-attempted in "
+ + (constraint.getOperationNextTime(candidateQuery) - System.currentTimeMillis())/1000 + " seconds";
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java b/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java
index 0a8d4c3..a7ee737 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/constraint/TotalQueryCostCeilingConstraint.java
@@ -57,18 +57,20 @@ public class TotalQueryCostCeilingConstraint implements QueryLaunchingConstraint
* @return
*/
@Override
- public boolean allowsLaunchOf(
+ public String allowsLaunchOf(
final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries) {
if (!totalQueryCostCeilingPerUser.isPresent()) {
- return true;
+ return null;
}
final String currentUser = candidateQuery.getSubmittedUser();
QueryCost totalQueryCostForCurrentUser = launchedQueries.getTotalQueryCost(currentUser);
- boolean canLaunch = (totalQueryCostForCurrentUser.compareTo(totalQueryCostCeilingPerUser.get()) <= 0);
- log.debug("canLaunch:{}", canLaunch);
- return canLaunch;
+ if (totalQueryCostForCurrentUser.compareTo(totalQueryCostCeilingPerUser.get()) > 0) {
+ return totalQueryCostForCurrentUser + "/" + totalQueryCostCeilingPerUser + " capacity utilized by "
+ + candidateQuery.getSubmittedUser();
+ }
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
index 18c2f2c..b2e140b 100644
--- a/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
+++ b/lens-server/src/main/java/org/apache/lens/server/rewrite/RewriteUtil.java
@@ -299,7 +299,7 @@ public final class RewriteUtil {
Map<LensDriver, DriverRewriterRunnable> runnables = new LinkedHashMap<>();
List<RewriteUtil.CubeQueryInfo> cubeQueries = findCubePositions(replacedQuery, ctx.getHiveConf());
- for (LensDriver driver : ctx.getDriverContext().getDrivers()) {
+ for (LensDriver driver : ctx.getDriverContext().getEligibleDrivers()) {
runnables.put(driver, new DriverRewriterRunnable(driver, ctx, cubeQueries, replacedQuery));
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerQueryEventListener.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerQueryEventListener.java b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerQueryEventListener.java
index 4192134..1c642bd 100644
--- a/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerQueryEventListener.java
+++ b/lens-server/src/main/java/org/apache/lens/server/scheduler/SchedulerQueryEventListener.java
@@ -25,7 +25,7 @@ import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.scheduler.*;
import org.apache.lens.server.api.events.AsyncEventListener;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.QueryEnded;
+import org.apache.lens.server.api.query.events.QueryEnded;
import lombok.extern.slf4j.Slf4j;
[4/4] lens git commit: LENS-743: Query retry framework for retrying
upon transient failures
Posted by pr...@apache.org.
LENS-743: Query retry framework for retrying upon transient failures
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/38ab6c60
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/38ab6c60
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/38ab6c60
Branch: refs/heads/master
Commit: 38ab6c6082b6221502daac979551e8c5fca72241
Parents: 182f6dc
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Mon Oct 17 11:13:34 2016 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Mon Oct 17 11:13:34 2016 +0530
----------------------------------------------------------------------
checkstyle/src/main/resources/checkstyle.xml | 2 +-
.../apache/lens/api/query/FailedAttempt.java | 68 ++++++
.../org/apache/lens/api/query/LensQuery.java | 6 +
.../org/apache/lens/api/query/QueryStatus.java | 34 ++-
.../lens/api/jaxb/YAMLToStringStrategyTest.java | 14 +-
.../org.apache.lens.api.query.LensQuery.xml | 5 -
.../org.apache.lens.api.query.QueryStatus.xml | 5 -
.../org/apache/lens/driver/es/ESDriver.java | 29 +--
lens-driver-es/src/test/resources/hive-site.xml | 5 +
.../org/apache/lens/driver/hive/HiveDriver.java | 65 +-----
.../apache/lens/driver/hive/TestHiveDriver.java | 2 +-
.../lens/driver/hive/TestRemoteHiveDriver.java | 2 +-
.../org/apache/lens/driver/jdbc/JDBCDriver.java | 69 ++----
.../driver/jdbc/JDBCDriverConfConstants.java | 6 -
.../jdbc/MaxJDBCConnectionCheckConstraint.java | 16 +-
.../apache/lens/driver/jdbc/TestJdbcDriver.java | 4 +-
.../regression/core/constants/DriverConfig.java | 4 +-
.../lens/server/api/LensConfConstants.java | 21 +-
.../server/api/common/BackOffRetryHandler.java | 71 -------
.../lens/server/api/common/FailureContext.java | 43 ----
...FibonacciExponentialBackOffRetryHandler.java | 77 -------
.../common/OperationRetryHandlerFactory.java | 41 ----
.../server/api/driver/AbstractLensDriver.java | 56 ++++-
.../server/api/driver/DriverConfiguration.java | 60 ++++++
.../server/api/driver/DriverQueryStatus.java | 48 ++---
.../lens/server/api/driver/LensDriver.java | 3 +
.../server/api/query/AbstractQueryContext.java | 2 +-
.../api/query/DriverSelectorQueryContext.java | 17 +-
.../server/api/query/FinishedLensQuery.java | 15 +-
.../lens/server/api/query/PriorityChange.java | 40 ----
.../lens/server/api/query/QueryAccepted.java | 39 ----
.../lens/server/api/query/QueryCancelled.java | 51 -----
.../lens/server/api/query/QueryClosed.java | 50 -----
.../lens/server/api/query/QueryContext.java | 68 ++++--
.../lens/server/api/query/QueryEnded.java | 87 --------
.../lens/server/api/query/QueryEvent.java | 93 --------
.../lens/server/api/query/QueryExecuted.java | 41 ----
.../lens/server/api/query/QueryFailed.java | 50 -----
.../lens/server/api/query/QueryLaunched.java | 41 ----
.../lens/server/api/query/QueryQueued.java | 59 ------
.../lens/server/api/query/QueryRejected.java | 39 ----
.../lens/server/api/query/QueryRunning.java | 41 ----
.../lens/server/api/query/QuerySuccess.java | 47 ----
.../server/api/query/QueuePositionChange.java | 39 ----
.../lens/server/api/query/StatusChange.java | 52 -----
.../api/query/StatusUpdateFailureContext.java | 45 ++++
.../query/comparators/ChainedComparator.java | 41 ++++
.../query/comparators/FIFOQueryComparator.java | 29 +++
.../comparators/MoreRetriesFirstComparator.java | 30 +++
.../api/query/comparators/QueryComparator.java | 28 +++
.../query/comparators/QueryCostComparator.java | 30 +++
.../comparators/QueryPriorityComparator.java | 29 +++
.../MaxConcurrentDriverQueriesConstraint.java | 64 ++++--
.../constraint/QueryLaunchingConstraint.java | 4 +-
.../server/api/query/events/PriorityChange.java | 40 ++++
.../server/api/query/events/QueryAccepted.java | 39 ++++
.../server/api/query/events/QueryCancelled.java | 52 +++++
.../server/api/query/events/QueryClosed.java | 51 +++++
.../server/api/query/events/QueryEnded.java | 88 ++++++++
.../server/api/query/events/QueryEvent.java | 93 ++++++++
.../server/api/query/events/QueryExecuted.java | 41 ++++
.../server/api/query/events/QueryFailed.java | 51 +++++
.../server/api/query/events/QueryLaunched.java | 41 ++++
.../server/api/query/events/QueryQueued.java | 59 ++++++
.../api/query/events/QueryQueuedForRetry.java | 41 ++++
.../server/api/query/events/QueryRejected.java | 39 ++++
.../server/api/query/events/QueryRunning.java | 41 ++++
.../server/api/query/events/QuerySuccess.java | 48 +++++
.../api/query/events/QueuePositionChange.java | 39 ++++
.../server/api/query/events/StatusChange.java | 52 +++++
.../server/api/retry/BackOffRetryHandler.java | 74 +++++++
.../api/retry/ChainedRetryPolicyDecider.java | 72 +++++++
.../api/retry/DefaultRetryPolicyDecider.java | 27 +++
.../lens/server/api/retry/FailureContext.java | 38 ++++
...FibonacciExponentialBackOffRetryHandler.java | 78 +++++++
.../server/api/retry/ImmediateRetryHandler.java | 46 ++++
.../lens/server/api/retry/NoRetryHandler.java | 35 +++
.../api/retry/OperationRetryHandlerFactory.java | 42 ++++
.../server/api/retry/RetryPolicyDecider.java | 25 +++
.../apache/lens/server/api/util/LensUtil.java | 34 +--
.../TestExponentialBackOffRetryHandler.java | 52 -----
.../lens/server/api/driver/MockDriver.java | 4 +-
.../lens/server/api/query/TestQueryContext.java | 3 +-
.../comparators/ChainedComparatorTest.java | 78 +++++++
...axConcurrentDriverQueriesConstraintTest.java | 43 +++-
.../TestExponentialBackOffRetryHandler.java | 55 +++++
lens-server/pom.xml | 2 +-
.../lens/server/metrics/MetricsServiceImpl.java | 2 +-
.../lens/server/query/FIFOQueryComparator.java | 33 ---
.../apache/lens/server/query/LensServerDAO.java | 98 +++++++--
.../lens/server/query/QueryComparator.java | 28 ---
.../lens/server/query/QueryCostComparator.java | 39 ----
.../lens/server/query/QueryEndHttpNotifier.java | 4 +-
.../lens/server/query/QueryEndNotifier.java | 2 +-
.../server/query/QueryEventHttpNotifier.java | 4 +-
.../server/query/QueryExecutionServiceImpl.java | 152 ++++++++++---
.../QueryExecutionStatisticsGenerator.java | 2 +-
.../server/query/QueryPriorityComparator.java | 38 ----
.../lens/server/query/ResultFormatter.java | 1 +
...DefaultQueryLaunchingConstraintsChecker.java | 19 +-
.../RetryPolicyToConstraingAdapter.java | 40 ++++
.../TotalQueryCostCeilingConstraint.java | 12 +-
.../apache/lens/server/rewrite/RewriteUtil.java | 2 +-
.../scheduler/SchedulerQueryEventListener.java | 2 +-
.../server/scheduler/SchedulerServiceImpl.java | 2 +-
.../query/QueryContextComparatorTest.java | 20 +-
.../lens/server/query/TestEventService.java | 2 +-
.../apache/lens/server/query/TestLensDAO.java | 23 +-
.../TestQueryIndependenceFromSessionClose.java | 5 +
.../query/TestQueryNotifictaionResource.java | 4 +-
.../lens/server/query/TestQueryService.java | 72 ++++---
.../server/query/collect/QueryCollectUtil.java | 4 +-
...ultQueryLaunchingConstraintsCheckerTest.java | 18 +-
.../ThreadSafeEstimatedQueryCollectionTest.java | 11 +-
.../TotalQueryCostCeilingConstraintTest.java | 18 +-
.../query/retry/MockDriverForRetries.java | 82 +++++++
.../query/retry/MockRetryPolicyDecider.java | 35 +++
.../lens/server/query/retry/QueryRetryTest.java | 212 +++++++++++++++++++
.../retry/TestServerRetryPolicyDecider.java | 32 +++
.../scheduler/util/SchedulerTestUtils.java | 2 +-
.../failing-query-driver-site.xml | 32 +++
.../retry/double_failure/driver-site.xml | 31 +++
.../retry/single_failure/driver-site.xml | 31 +++
.../retry/triple_failure/driver-site.xml | 31 +++
124 files changed, 2988 insertions(+), 1607 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/checkstyle/src/main/resources/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml
index 17e55ea..634234d 100644
--- a/checkstyle/src/main/resources/checkstyle.xml
+++ b/checkstyle/src/main/resources/checkstyle.xml
@@ -62,7 +62,7 @@
</module>
<module name="FileLength">
- <property name="max" value="3500"/>
+ <property name="max" value="4000"/>
</module>
<module name="FileTabCharacter"/>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java b/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java
new file mode 100644
index 0000000..5cf2dc3
--- /dev/null
+++ b/lens-api/src/main/java/org/apache/lens/api/query/FailedAttempt.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.api.query;
+
+import java.io.Serializable;
+
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import lombok.*;
+
+
+@XmlRootElement
+@AllArgsConstructor
+/**
+ * Instantiates a new query status.
+ */
+@NoArgsConstructor(access = AccessLevel.PROTECTED)
+@EqualsAndHashCode
+public class FailedAttempt implements Comparable<FailedAttempt>, Serializable {
+ @XmlElement
+ @Getter
+ private String driverName;
+ @XmlElement
+ @Getter
+ private double progress;
+ @XmlElement
+ @Getter
+ private String progressMessage;
+ @XmlElement
+ @Getter
+ private String errorMessage;
+ @XmlElement
+ @Getter
+ private Long driverStartTime;
+ @XmlElement
+ @Getter
+ private Long driverFinishTime;
+
+ @Override
+ public int compareTo(FailedAttempt o) {
+ if (o == null) {
+ return 1;
+ }
+ int startCompare = driverStartTime.compareTo(o.getDriverStartTime());
+ if (startCompare != 0) {
+ return startCompare;
+ }
+ return driverFinishTime.compareTo(o.getDriverFinishTime());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
index 7d079a9..d7b8963 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/LensQuery.java
@@ -21,6 +21,8 @@
*/
package org.apache.lens.api.query;
+import java.util.List;
+
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
@@ -215,6 +217,10 @@ public class LensQuery extends ToYAMLString {
@Getter
private String queryName;
+ @XmlElement
+ @Getter
+ private List<FailedAttempt> failedAttempts;
+
/**
* @return error code in case of query failures
*/
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
index daedbf9..3c6ecf7 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/QueryStatus.java
@@ -62,11 +62,11 @@ import lombok.*;
@NoArgsConstructor(access = AccessLevel.PROTECTED)
public class QueryStatus extends ToYAMLString implements Serializable {
+
/**
* The Constant serialVersionUID.
*/
private static final long serialVersionUID = 1L;
-
/**
* The Enum Status.
*/
@@ -106,6 +106,12 @@ public class QueryStatus extends ToYAMLString implements Serializable {
EXECUTED,
/**
+ * This state is when depending on retry policy, either the query moves to QUEUED (in case retries are to be done),
+ * or to FAILED.
+ */
+ FAILING,
+
+ /**
* The successful.
* At this point all operations related to the query are finished successfully by driver and server.
*/
@@ -177,7 +183,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
@Setter
private String errorMessage;
- @XmlElement
+ @Getter
private LensErrorTO lensErrorTO;
public boolean finished() {
@@ -205,6 +211,10 @@ public class QueryStatus extends ToYAMLString implements Serializable {
return status.equals(Status.FAILED);
}
+ public boolean failing() {
+ return status.equals(Status.FAILING);
+ }
+
public boolean cancelled() {
return status.equals(Status.CANCELED);
}
@@ -232,7 +242,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
case QUEUED:
switch (newState) {
case LAUNCHED:
- case FAILED:
+ case FAILING:
case CANCELED:
return true;
}
@@ -242,7 +252,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
case LAUNCHED:
case RUNNING:
case CANCELED:
- case FAILED:
+ case FAILING:
case EXECUTED:
return true;
}
@@ -251,7 +261,7 @@ public class QueryStatus extends ToYAMLString implements Serializable {
switch (newState) {
case RUNNING:
case CANCELED:
- case FAILED:
+ case FAILING:
case EXECUTED:
return true;
}
@@ -260,17 +270,25 @@ public class QueryStatus extends ToYAMLString implements Serializable {
switch (newState) {
case EXECUTED:
case SUCCESSFUL:
- case FAILED:
+ case FAILING:
case CANCELED:
return true;
}
break;
+ case FAILING:
+ switch(newState) {
+ case QUEUED:
+ case FAILED:
+ return true;
+ }
+ break;
case FAILED:
case CANCELED:
case SUCCESSFUL:
if (Status.CLOSED.equals(newState)) {
return true;
}
+ break;
default:
// fall-through
}
@@ -294,4 +312,8 @@ public class QueryStatus extends ToYAMLString implements Serializable {
public String getLensErrorTOErrorMsg() {
return (this.lensErrorTO != null) ? this.lensErrorTO.getMessage() : null;
}
+
+ public static QueryStatus getQueuedStatus() {
+ return new QueryStatus(0.0, null, Status.QUEUED, "Query is queued", false, null, null, null);
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
----------------------------------------------------------------------
diff --git a/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java b/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
index 04d9ad6..eb568a7 100644
--- a/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
+++ b/lens-api/src/test/java/org/apache/lens/api/jaxb/YAMLToStringStrategyTest.java
@@ -147,11 +147,15 @@ public class YAMLToStringStrategyTest {
return name.endsWith("xml");
}
})) {
- Class<?> clazz = Class.forName(fn.substring(0, fn.length() - 4));
- Object unmarshalled = clazz.cast(new LensJAXBContext(clazz)
- .createUnmarshaller().unmarshal(getClass().getResourceAsStream("/toString/" + fn)));
- String toString = readYAML("/toString/" + fn.replaceAll("xml$", "yaml"));
- dataList.add(new ToStringTestData(fn, unmarshalled, toString));
+ try {
+ Class<?> clazz = Class.forName(fn.substring(0, fn.length() - 4));
+ Object unmarshalled = clazz.cast(new LensJAXBContext(clazz)
+ .createUnmarshaller().unmarshal(getClass().getResourceAsStream("/toString/" + fn)));
+ String toString = readYAML("/toString/" + fn.replaceAll("xml$", "yaml"));
+ dataList.add(new ToStringTestData(fn, unmarshalled, toString));
+ } catch (Throwable th) {
+ dataList.add(new ToStringTestData(fn, th));
+ }
}
return dataList;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
----------------------------------------------------------------------
diff --git a/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml b/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
index a681bca..fc609d9 100644
--- a/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
+++ b/lens-api/src/test/resources/toString/org.apache.lens.api.query.LensQuery.xml
@@ -34,11 +34,6 @@
<status>RUNNING</status>
<statusMessage>query running</statusMessage>
<isResultSetAvailable>false</isResultSetAvailable>
- <lensErrorTO>
- <code>9999</code>
- <message>error message</message>
- <stackTrace>random stack trace</stackTrace>
- </lensErrorTO>
<errorMessage>error message</errorMessage>
<progressMessage>progress message</progressMessage>
<queueNumber>11</queueNumber>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
----------------------------------------------------------------------
diff --git a/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml b/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
index 70f9c93..b1957d7 100644
--- a/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
+++ b/lens-api/src/test/resources/toString/org.apache.lens.api.query.QueryStatus.xml
@@ -24,11 +24,6 @@
<status>RUNNING</status>
<statusMessage>query running</statusMessage>
<isResultSetAvailable>false</isResultSetAvailable>
- <lensErrorTO>
- <code>9999</code>
- <message>error message</message>
- <stackTrace>random stack trace</stackTrace>
- </lensErrorTO>
<errorMessage>error message</errorMessage>
<progressMessage>progress message</progressMessage>
<queueNumber>11</queueNumber>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
index fceabea..e735c0c 100644
--- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/ESDriver.java
@@ -42,8 +42,6 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
-import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
@@ -60,9 +58,7 @@ import org.apache.hadoop.hive.ql.session.SessionState;
import org.antlr.runtime.CommonToken;
import org.antlr.runtime.tree.Tree;
-import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
import lombok.extern.slf4j.Slf4j;
@@ -76,7 +72,6 @@ public class ESDriver extends AbstractLensDriver {
private static final double STREAMING_PARTITION_COST = 0;
private static final QueryCost ES_DRIVER_COST = new FactPartitionBasedQueryCost(STREAMING_PARTITION_COST);
- private Configuration conf;
private ESClient esClient;
private ExecutorService asyncQueryPool;
private ESDriverConfig config;
@@ -89,11 +84,6 @@ public class ESDriver extends AbstractLensDriver {
private final Map<QueryHandle, QueryContext> handleContextMap = Maps.newConcurrentMap();
@Override
- public Configuration getConf() {
- return conf;
- }
-
- @Override
public QueryCost estimate(AbstractQueryContext qctx) {
return ES_DRIVER_COST;
}
@@ -232,16 +222,6 @@ public class ESDriver extends AbstractLensDriver {
}
- @Override
- public ImmutableSet<QueryLaunchingConstraint> getQueryConstraints() {
- return ImmutableSet.copyOf(Sets.<QueryLaunchingConstraint>newHashSet());
- }
-
- @Override
- public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
- return ImmutableSet.copyOf(Sets.<WaitingQueriesSelectionPolicy>newHashSet());
- }
-
private ESQuery rewrite(AbstractQueryContext context) throws LensException {
final String key = keyFor(context);
if (rewrittenQueriesCache.containsKey(key)) {
@@ -299,18 +279,15 @@ public class ESDriver extends AbstractLensDriver {
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
super.configure(conf, driverType, driverName);
- this.conf = new Configuration(conf);
- this.conf.addResource("esdriver-default.xml");
- this.conf.addResource(getDriverResourcePath("esdriver-site.xml"));
- config = new ESDriverConfig(this.conf);
+ config = new ESDriverConfig(getConf());
Class klass;
try {
- klass = Class.forName(this.conf.get(ESDriverConfig.CLIENT_CLASS_KEY));
+ klass = Class.forName(getConf().get(ESDriverConfig.CLIENT_CLASS_KEY));
if (klass != null) {
log.debug("Picked up class {}", klass);
if (ESClient.class.isAssignableFrom(klass)) {
final Constructor constructor = klass.getConstructor(ESDriverConfig.class, Configuration.class);
- esClient = (ESClient) constructor.newInstance(config, this.conf);
+ esClient = (ESClient) constructor.newInstance(config, getConf());
log.debug("Successfully instantiated es client of type {}", klass);
}
} else {
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-es/src/test/resources/hive-site.xml
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/test/resources/hive-site.xml b/lens-driver-es/src/test/resources/hive-site.xml
index e49c876..f2f4b9b 100644
--- a/lens-driver-es/src/test/resources/hive-site.xml
+++ b/lens-driver-es/src/test/resources/hive-site.xml
@@ -50,4 +50,9 @@
<value>true</value>
</property>
+ <property>
+ <name>hive.metastore.schema.verification</name>
+ <value>false</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 0d8810f..f307ccc 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -19,7 +19,6 @@
package org.apache.lens.driver.hive;
import static org.apache.lens.server.api.error.LensDriverErrorCode.*;
-import static org.apache.lens.server.api.util.LensUtil.getImplementations;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@@ -45,8 +44,6 @@ import org.apache.lens.server.api.events.LensEventListener;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
-import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.cost.QueryCostCalculator;
@@ -70,7 +67,6 @@ import org.apache.hive.service.rpc.thrift.TSessionHandle;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
-import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -83,8 +79,6 @@ public class HiveDriver extends AbstractLensDriver {
/** The Constant HIVE_CONNECTION_CLASS. */
public static final String HIVE_CONNECTION_CLASS = "lens.driver.hive.connection.class";
- public static final String HIVE_QUERY_HOOK_CLASS = "lens.driver.hive.query.hook.class";
-
/** The Constant HS2_CONNECTION_EXPIRY_DELAY. */
public static final String HS2_CONNECTION_EXPIRY_DELAY = "lens.driver.hive.hs2.connection.expiry.delay";
@@ -101,15 +95,6 @@ public class HiveDriver extends AbstractLensDriver {
public static final String HS2_PRIORITY_DEFAULT_RANGES = "VERY_HIGH,7.0,HIGH,30.0,NORMAL,90,LOW";
public static final String SESSION_KEY_DELIMITER = ".";
- public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY
- = "lens.driver.hive.query.launching.constraint.factories";
-
- private static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY
- = "lens.driver.hive.waiting.queries.selection.policy.factories";
-
- /** The driver conf- which will merged with query conf */
- private Configuration driverConf;
-
/** The HiveConf - used for connecting to hive server and metastore */
private HiveConf hiveConf;
@@ -153,17 +138,12 @@ public class HiveDriver extends AbstractLensDriver {
QueryPriorityDecider queryPriorityDecider;
// package-local. Test case can change.
boolean whetherCalculatePriority;
- private DriverQueryHook queryHook;
private static final Map<String, String> SESSION_CONF = new HashMap<String, String>() {
{
put(HiveConf.ConfVars.HIVE_SERVER2_CLOSE_SESSION_ON_DISCONNECT.varname, "false");
}
};
- @Getter
- protected ImmutableSet<QueryLaunchingConstraint> queryConstraints;
- private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies;
-
private String sessionDbKey(String sessionHandle, String database) {
return sessionHandle + SESSION_KEY_DELIMITER + database;
}
@@ -340,11 +320,6 @@ public class HiveDriver extends AbstractLensDriver {
log.info("Hive driver inited");
}
- @Override
- public Configuration getConf() {
- return driverConf;
- }
-
/*
* (non-Javadoc)
*
@@ -353,23 +328,16 @@ public class HiveDriver extends AbstractLensDriver {
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
super.configure(conf, driverType, driverName);
- this.driverConf = new Configuration(conf);
- String driverConfPath = getDriverResourcePath("hivedriver-site.xml");
- this.driverConf.addResource("hivedriver-default.xml");
- this.driverConf.addResource(driverConfPath);
- // resources have to be added separately on hiveConf again because new HiveConf() overrides hive.* properties
- // from HiveConf
this.hiveConf = new HiveConf(conf, HiveDriver.class);
- this.hiveConf.addResource("hivedriver-default.xml");
- this.hiveConf.addResource(driverConfPath);
+ this.hiveConf.addResource(getConf());
- connectionClass = this.driverConf.getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class,
+ connectionClass = getConf().getClass(HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class,
ThriftConnection.class);
isEmbedded = (connectionClass.getName().equals(EmbeddedThriftConnection.class.getName()));
- connectionExpiryTimeout = this.driverConf.getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
- whetherCalculatePriority = this.driverConf.getBoolean(HS2_CALCULATE_PRIORITY, true);
- Class<? extends QueryCostCalculator> queryCostCalculatorClass = this.driverConf.getClass(HS2_COST_CALCULATOR,
+ connectionExpiryTimeout = getConf().getLong(HS2_CONNECTION_EXPIRY_DELAY, DEFAULT_EXPIRY_DELAY);
+ whetherCalculatePriority = getConf().getBoolean(HS2_CALCULATE_PRIORITY, true);
+ Class<? extends QueryCostCalculator> queryCostCalculatorClass = getConf().getClass(HS2_COST_CALCULATOR,
FactPartitionBasedQueryCostCalculator.class, QueryCostCalculator.class);
try {
queryCostCalculator = queryCostCalculatorClass.newInstance();
@@ -377,18 +345,9 @@ public class HiveDriver extends AbstractLensDriver {
throw new LensException("Can't instantiate query cost calculator of class: " + queryCostCalculatorClass, e);
}
queryPriorityDecider = new CostRangePriorityDecider(
- new CostToPriorityRangeConf(driverConf.get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES))
+ new CostToPriorityRangeConf(getConf().get(HS2_PRIORITY_RANGES, HS2_PRIORITY_DEFAULT_RANGES))
);
- try {
- queryHook = driverConf.getClass(
- HIVE_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
- ).newInstance();
- queryHook.setDriver(this);
- } catch (InstantiationException | IllegalAccessException e) {
- throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
- }
- queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, driverConf);
- selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, driverConf);
+
log.info("Hive driver {} configured successfully", getFullyQualifiedName());
}
@@ -809,11 +768,6 @@ public class HiveDriver extends AbstractLensDriver {
}
@Override
- public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
- return selectionPolicies;
- }
-
- @Override
public Priority decidePriority(AbstractQueryContext ctx) {
return decidePriority(ctx, queryPriorityDecider);
}
@@ -1385,9 +1339,4 @@ public class HiveDriver extends AbstractLensDriver {
public boolean hasLensSession(LensSessionHandle session) {
return lensToHiveSession.containsKey(session.getPublicId().toString());
}
-
- @Override
- public DriverQueryHook getQueryHook() {
- return queryHook;
- }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
index daf01ac..43b33f3 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestHiveDriver.java
@@ -118,7 +118,7 @@ public class TestHiveDriver {
protected void createDriver() throws LensException {
driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driverConf.setClass(HiveDriver.HIVE_CONNECTION_CLASS, EmbeddedThriftConnection.class, ThriftConnection.class);
- driverConf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
+ driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
driverConf.set("hive.lock.manager", "org.apache.hadoop.hive.ql.lockmgr.EmbeddedLockManager");
driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
driver = new HiveDriver();
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
index 1acbb13..961ec4e 100644
--- a/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
+++ b/lens-driver-hive/src/test/java/org/apache/lens/driver/hive/TestRemoteHiveDriver.java
@@ -148,7 +148,7 @@ public class TestRemoteHiveDriver extends TestHiveDriver {
driverConf.addResource("drivers/hive/hive1/hivedriver-site.xml");
driver = new HiveDriver();
driverConf.setBoolean(HiveDriver.HS2_CALCULATE_PRIORITY, true);
- driverConf.setClass(HiveDriver.HIVE_QUERY_HOOK_CLASS, MockDriverQueryHook.class, DriverQueryHook.class);
+ driverConf.setClass(LensConfConstants.DRIVER_HOOK_CLASS_SFX, MockDriverQueryHook.class, DriverQueryHook.class);
driver.configure(driverConf, "hive", "hive1");
drivers = Lists.<LensDriver>newArrayList(driver);
System.out.println("TestRemoteHiveDriver created");
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
index e41077c..82e0231 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriver.java
@@ -23,7 +23,6 @@ import static java.util.Arrays.asList;
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.*;
import static org.apache.lens.driver.jdbc.JDBCDriverConfConstants.ConnectionPoolProperties.*;
-import static org.apache.lens.server.api.util.LensUtil.getImplementations;
import static com.google.common.base.Preconditions.checkState;
@@ -51,9 +50,7 @@ import org.apache.lens.server.api.metrics.MethodMetricsFactory;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.PreparedQueryContext;
import org.apache.lens.server.api.query.QueryContext;
-import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
-import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
import org.apache.lens.server.api.query.cost.QueryCost;
import org.apache.lens.server.api.query.rewrite.QueryRewriter;
@@ -68,8 +65,6 @@ import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.parse.ASTNode;
import org.apache.hadoop.hive.ql.parse.HiveParser;
-import com.google.common.collect.ImmutableSet;
-
import lombok.*;
import lombok.extern.slf4j.Slf4j;
@@ -95,20 +90,12 @@ public class JDBCDriver extends AbstractLensDriver {
@Getter
private ConcurrentHashMap<QueryHandle, JdbcQueryContext> queryContextMap;
- /** The conf. */
- private Configuration conf;
-
/** Configuration for estimate connection pool */
private Configuration estimateConf;
/** Estimate connection provider */
private ConnectionProvider estimateConnectionProvider;
private LogSegregationContext logSegregationContext;
- private DriverQueryHook queryHook;
-
- @Getter
- private ImmutableSet<QueryLaunchingConstraint> queryConstraints;
- private ImmutableSet<WaitingQueriesSelectionPolicy> selectionPolicies;
private boolean isStatementCancelSupported;
/**
@@ -321,6 +308,7 @@ public class JDBCDriver extends AbstractLensDriver {
queryContext.getLensContext().getDriverStatus().setDriverFinishTime(System.currentTimeMillis());
}
}
+
return result;
}
@@ -383,14 +371,6 @@ public class JDBCDriver extends AbstractLensDriver {
}
}
- /**
- * Get driver configuration
- */
- @Override
- public Configuration getConf() {
- return conf;
- }
-
/*
* (non-Javadoc)
*
@@ -399,18 +379,7 @@ public class JDBCDriver extends AbstractLensDriver {
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
super.configure(conf, driverType, driverName);
- this.conf = new Configuration(conf);
- this.conf.addResource("jdbcdriver-default.xml");
- this.conf.addResource(getDriverResourcePath("jdbcdriver-site.xml"));
- init(conf);
- try {
- queryHook = this.conf.getClass(
- JDBC_QUERY_HOOK_CLASS, NoOpDriverQueryHook.class, DriverQueryHook.class
- ).newInstance();
- queryHook.setDriver(this);
- } catch (InstantiationException | IllegalAccessException e) {
- throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
- }
+ init();
configured = true;
log.info("JDBC Driver {} configured", getFullyQualifiedName());
}
@@ -418,14 +387,12 @@ public class JDBCDriver extends AbstractLensDriver {
/**
* Inits the.
*
- * @param conf the conf
* @throws LensException the lens exception
*/
- protected void init(Configuration conf) throws LensException {
-
- final int maxPoolSize = parseInt(this.conf.get(JDBC_POOL_MAX_SIZE.getConfigKey()));
+ public void init() throws LensException {
+ final int maxPoolSize = parseInt(getConf().get(JDBC_POOL_MAX_SIZE.getConfigKey()));
final int maxConcurrentQueries
- = parseInt(this.conf.get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY));
+ = parseInt(getConf().get(MaxConcurrentDriverQueriesConstraintFactory.MAX_CONCURRENT_QUERIES_KEY));
checkState(maxPoolSize >= maxConcurrentQueries, "maxPoolSize:" + maxPoolSize + " maxConcurrentQueries:"
+ maxConcurrentQueries);
@@ -439,7 +406,7 @@ public class JDBCDriver extends AbstractLensDriver {
}
});
- Class<? extends ConnectionProvider> cpClass = conf.getClass(JDBC_CONNECTION_PROVIDER,
+ Class<? extends ConnectionProvider> cpClass = getConf().getClass(JDBC_CONNECTION_PROVIDER,
DataSourceConnectionProvider.class, ConnectionProvider.class);
try {
connectionProvider = cpClass.newInstance();
@@ -449,9 +416,8 @@ public class JDBCDriver extends AbstractLensDriver {
throw new LensException(e);
}
this.logSegregationContext = new MappedDiagnosticLogSegregationContext();
- this.queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY, this.conf);
- this.selectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY, this.conf);
- this.isStatementCancelSupported = conf.getBoolean(STATEMENT_CANCEL_SUPPORTED, DEFAULT_STATEMENT_CANCEL_SUPPORTED);
+ this.isStatementCancelSupported = getConf().getBoolean(STATEMENT_CANCEL_SUPPORTED,
+ DEFAULT_STATEMENT_CANCEL_SUPPORTED);
}
/**
@@ -469,7 +435,7 @@ public class JDBCDriver extends AbstractLensDriver {
try {
// Add here to cover the path when the queries are executed it does not
// use the driver conf
- return connectionProvider.getConnection(conf);
+ return connectionProvider.getConnection(getConf());
} catch (SQLException e) {
throw new LensException(e);
}
@@ -483,7 +449,7 @@ public class JDBCDriver extends AbstractLensDriver {
*/
protected QueryRewriter getQueryRewriter() throws LensException {
QueryRewriter rewriter;
- Class<? extends QueryRewriter> queryRewriterClass = conf.getClass(JDBC_QUERY_REWRITER_CLASS,
+ Class<? extends QueryRewriter> queryRewriterClass = getConf().getClass(JDBC_QUERY_REWRITER_CLASS,
DummyQueryRewriter.class, QueryRewriter.class);
try {
rewriter = queryRewriterClass.newInstance();
@@ -492,7 +458,7 @@ public class JDBCDriver extends AbstractLensDriver {
log.error("{} Unable to create rewriter object", getFullyQualifiedName(), e);
throw new LensException(e);
}
- rewriter.init(conf);
+ rewriter.init(getConf());
return rewriter;
}
@@ -674,7 +640,7 @@ public class JDBCDriver extends AbstractLensDriver {
// Get connection config used by estimate pool.
protected final Configuration getEstimateConnectionConf() {
if (estimateConf == null) {
- Configuration tmpConf = new Configuration(conf);
+ Configuration tmpConf = new Configuration(getConf());
// Override JDBC settings in estimate conf, if set by user explicitly. Otherwise fall back to default JDBC pool
// config
for (String key : asList(JDBC_CONNECTION_PROPERTIES, JDBC_DB_URI, JDBC_DRIVER_CLASS, JDBC_USER, JDBC_PASSWORD,
@@ -1059,11 +1025,6 @@ public class JDBCDriver extends AbstractLensDriver {
}
- @Override
- public ImmutableSet<WaitingQueriesSelectionPolicy> getWaitingQuerySelectionPolicies() {
- return this.selectionPolicies;
- }
-
/*
* (non-Javadoc)
*
@@ -1084,12 +1045,6 @@ public class JDBCDriver extends AbstractLensDriver {
public void writeExternal(ObjectOutput arg0) throws IOException {
// TODO Auto-generated method stub
}
-
- @Override
- public DriverQueryHook getQueryHook() {
- return queryHook;
- }
-
@Override
public StatusUpdateMethod getStatusUpdateMethod() {
return StatusUpdateMethod.PUSH;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
index 3c34eb1..f4e0451 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCDriverConfConstants.java
@@ -38,7 +38,6 @@ public final class JDBCDriverConfConstants {
/** The Constant JDBC_QUERY_REWRITER_CLASS. */
public static final String JDBC_QUERY_REWRITER_CLASS = JDBC_DRIVER_PFX + "query.rewriter";
- public static final String JDBC_QUERY_HOOK_CLASS = JDBC_DRIVER_PFX + "query.hook.class";
/** The Constant JDBC_DRIVER_CLASS. */
public static final String JDBC_DRIVER_CLASS = JDBC_DRIVER_PFX + "driver.class";
@@ -104,11 +103,6 @@ public final class JDBCDriverConfConstants {
public static final String JDBC_FETCH_SIZE = JDBC_DRIVER_PFX + "fetch.size";
public static final int DEFAULT_JDBC_FETCH_SIZE = 1000;
- public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = JDBC_DRIVER_PFX
- + "query.launching.constraint.factories";
-
- public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = JDBC_DRIVER_PFX
- + "waiting.queries.selection.policy.factories";
public static final String REGEX_REPLACEMENT_VALUES = JDBC_DRIVER_PFX + "regex.replacement.values";
public static final String STATEMENT_CANCEL_SUPPORTED = JDBC_DRIVER_PFX + "statement.cancel.supported";
public static final boolean DEFAULT_STATEMENT_CANCEL_SUPPORTED = true;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
index 82b5647..f293ea8 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/MaxJDBCConnectionCheckConstraint.java
@@ -35,14 +35,18 @@ public class MaxJDBCConnectionCheckConstraint implements QueryLaunchingConstrain
}
@Override
- public boolean allowsLaunchOf(final QueryContext candidateQuery,
+ public String allowsLaunchOf(final QueryContext candidateQuery,
EstimatedImmutableQueryCollection launchedQueries) {
final LensDriver selectedDriver = candidateQuery.getSelectedDriver();
- final boolean canLaunch = (selectedDriver instanceof JDBCDriver)
- && (((JDBCDriver) selectedDriver).getQueryContextMap().size() < poolMaxSize);
-
- log.debug("canLaunch:{}", canLaunch);
- return canLaunch;
+ if (!(selectedDriver instanceof JDBCDriver)) {
+ return "driver isn't jdbc driver";
+ }
+ int runningQueries = ((JDBCDriver) selectedDriver).getQueryContextMap().size();
+ if (runningQueries >= poolMaxSize) {
+ return runningQueries + "/" + poolMaxSize + " queries running on driver "
+ + candidateQuery.getSelectedDriver().getFullyQualifiedName();
+ }
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
index 2ad7f76..a402f91 100644
--- a/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
+++ b/lens-driver-jdbc/src/test/java/org/apache/lens/driver/jdbc/TestJdbcDriver.java
@@ -446,11 +446,11 @@ public class TestJdbcDriver {
//new query shouldn't be allowed
QueryContext newcontext = createQueryContext("SELECT 123 FROM max_connection_test");
- assertFalse(constraint.allowsLaunchOf(newcontext, null));
+ assertNotNull(constraint.allowsLaunchOf(newcontext, null));
//close one query and launch the previous query again
driver.closeQuery(context.getQueryHandle());
- assertTrue(constraint.allowsLaunchOf(newcontext, null));
+ assertNull(constraint.allowsLaunchOf(newcontext, null));
close();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
----------------------------------------------------------------------
diff --git a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
index d80fba9..cff8e91 100644
--- a/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
+++ b/lens-regression/src/main/java/org/apache/lens/regression/core/constants/DriverConfig.java
@@ -19,8 +19,8 @@
package org.apache.lens.regression.core.constants;
-import org.apache.lens.driver.hive.HiveDriver;
import org.apache.lens.driver.jdbc.JDBCDriverConfConstants;
+import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.query.constraint.MaxConcurrentDriverQueriesConstraintFactory;
import org.apache.lens.server.query.constraint.TotalQueryCostCeilingConstraintFactory;
@@ -38,7 +38,7 @@ public class DriverConfig {
MAX_CONCURRENT_QUERIES_PER_QUEUE_KEY;
public static final String JDBC_POOL_SIZE = JDBCDriverConfConstants.ConnectionPoolProperties.
JDBC_POOL_MAX_SIZE.getConfigKey();
- public static final String HIVE_CONSTRAINT_FACTORIES = HiveDriver.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY;
+ public static final String HIVE_CONSTRAINT_FACTORIES = LensConfConstants.QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX;
public static final String MAX_CONCURRENT_CONSTRAINT_FACTORY = MaxConcurrentDriverQueriesConstraintFactory
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
index 8cf617b..3ae59c6 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/LensConfConstants.java
@@ -98,7 +98,7 @@ public final class LensConfConstants {
public static final String MAX_SESSIONS_PER_USER = SERVER_PFX + "max.sessions.per.user";
- public static final String QUERY_COMPARATOR_CLASS = SERVER_PFX + "query.comparator.class";
+ public static final String QUERY_COMPARATOR_CLASSES = SERVER_PFX + "query.comparator.classes";
public static final Integer DEFAULT_MAX_SESSIONS_PER_USER = 10;
@@ -913,8 +913,9 @@ public final class LensConfConstants {
/**
* Key to get the implementations of query constraint factories.
*/
+ public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX = "query.launching.constraint.factories";
public static final String QUERY_LAUNCHING_CONSTRAINT_FACTORIES_KEY = SERVER_PFX
- + "query.launching.constraint.factories";
+ + QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX;
/**
* Key to get the total query cost ceiling per user.
@@ -925,8 +926,10 @@ public final class LensConfConstants {
/**
* Key to get the implementations of waiting queries selection policy factories.
*/
+ public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX =
+ "waiting.queries.selection.policy.factories";
public static final String WAITING_QUERIES_SELECTION_POLICY_FACTORIES_KEY = SERVER_PFX
- + "waiting.queries.selection.policy.factories";
+ + WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX;
/**
* Key denoting the dialect class property of saved query service.
@@ -954,6 +957,18 @@ public final class LensConfConstants {
public static final String DRIVER_WEIGHT = DRIVER_PFX + "weight";
/**
+ * Key for specifying Retry policy class
+ */
+ public static final String RETRY_POLICY_CLASSES_SFX = "query.retry.policy.classes";
+
+ public static final String QUERY_RETRY_POLICY_CLASSES = SERVER_PFX + RETRY_POLICY_CLASSES_SFX;
+
+ /**
+ * Driver hook property
+ */
+ public static final String DRIVER_HOOK_CLASS_SFX = "query.hook.class";
+
+ /**
* Default driver weight
*/
public static final int DEFAULT_DRIVER_WEIGHT = 1;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
deleted file mode 100644
index 17bfba0..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/BackOffRetryHandler.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-/**
- * A backoff retry handler.
- *
- * This allows a backoff on any call, so provides methods whether we can try the operation now,
- * whats next time when operation can be performed and whether operation has exhausted all retries.
- *
- * Callers of this would do the following :
- *
- * if (handler.canTryOpNow(FailureContext)) {
- * try {
- * tryCallerOperation();
- * FailureContext.clear();
- * } catch (any Transient Exception) {
- * FailureContext.updateFailure();
- * if (!handler.hasExhaustedRetries(FailureContext)) {
- * // will be tried later again
- * }
- * throw exception;
- * }
- * }
- */
-public interface BackOffRetryHandler {
-
- /**
- * To know whether operation can be done now.
- *
- * @param failContext FailureContext holding failures till now.
- *
- * @return true if operation can be done now, false otherwise.
- */
- boolean canTryOpNow(FailureContext failContext);
-
- /**
- * Get the time when the operation can be done next.
- *
- * @param failContext FailureContext holding failures till now.
- *
- * @return Next operation time in millis since epoch
- */
- long getOperationNextTime(FailureContext failContext);
-
- /**
- * Has the operation exhausted all its retries
- *
- * @param failContext FailureContext holding failures till now.
- *
- * @return true if all retries have exhausted, false otherwise.
- */
- boolean hasExhaustedRetries(FailureContext failContext);
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java
deleted file mode 100644
index 70a34b0..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FailureContext.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-import lombok.Getter;
-
-/**
- * Failure context captures last failure time and number of failures.
- */
-public class FailureContext {
-
- @Getter
- private long lastFailedTime = 0;
- @Getter
- private int failCount = 0;
-
- public synchronized void updateFailure() {
- lastFailedTime = System.currentTimeMillis();
- failCount++;
- }
-
- public synchronized void clear() {
- lastFailedTime = 0;
- failCount = 0;
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java
deleted file mode 100644
index e7fb8ce..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/FibonacciExponentialBackOffRetryHandler.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-/**
- * A exponential backoff retry handler.
- *
- * It allows the the failures to be retried at a next update time, which can increase exponentially.
- *
- */
-public class FibonacciExponentialBackOffRetryHandler implements BackOffRetryHandler {
- final int[] fibonacci;
- final long maxDelay;
- final long waitMillis;
-
- public FibonacciExponentialBackOffRetryHandler(int numRetries, long maxDelay, long waitMillis) {
- checkArgument(numRetries > 2);
- fibonacci = new int[numRetries];
- fibonacci[0] = 1;
- fibonacci[1] = 1;
- for(int i = 2; i < numRetries; ++i) {
- fibonacci[i] = fibonacci[i-1] + fibonacci[i-2];
- }
- this.maxDelay = maxDelay;
- this.waitMillis = waitMillis;
- }
-
- public boolean canTryOpNow(FailureContext failContext) {
- synchronized (failContext) {
- if (failContext.getFailCount() != 0) {
- long now = System.currentTimeMillis();
- if (now < getOperationNextTime(failContext)) {
- return false;
- }
- }
- return true;
- }
- }
-
- public long getOperationNextTime(FailureContext failContext) {
- synchronized (failContext) {
- if (failContext.getFailCount() >= fibonacci.length) {
- return failContext.getLastFailedTime() + maxDelay;
- }
- long delay = Math.min(maxDelay, fibonacci[failContext.getFailCount()] * waitMillis);
- return failContext.getLastFailedTime() + delay;
- }
- }
-
- public boolean hasExhaustedRetries(FailureContext failContext) {
- synchronized (failContext) {
- if (failContext.getFailCount() >= fibonacci.length) {
- return true;
- }
- return false;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java b/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java
deleted file mode 100644
index 88fbe43..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/common/OperationRetryHandlerFactory.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.lens.server.api.common;
-
-/**
- * Factory which creates operation retry handler
- */
-public class OperationRetryHandlerFactory {
- private OperationRetryHandlerFactory() {
- }
-
- /**
- * Create exponential backoff handler
- *
- * @param numRetries Number of exponential backoff retries
- * @param maxDelay Maximum delay an operation can wait for next
- * @param waitMillis Number of millis that would grow exponentially incase of failures
- *
- * @return BackOffRetryHandler
- */
- public static BackOffRetryHandler createExponentialBackOffHandler(int numRetries, long maxDelay, long waitMillis) {
- return new FibonacciExponentialBackOffRetryHandler(numRetries, maxDelay, waitMillis);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
index 365a619..8f30aa0 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/AbstractLensDriver.java
@@ -18,15 +18,23 @@
*/
package org.apache.lens.server.api.driver;
+import static org.apache.lens.server.api.LensConfConstants.*;
+import static org.apache.lens.server.api.util.LensUtil.getImplementations;
+
import org.apache.lens.api.Priority;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.AbstractQueryContext;
import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
+import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.retry.ChainedRetryPolicyDecider;
+import org.apache.lens.server.api.retry.RetryPolicyDecider;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import com.google.common.collect.ImmutableSet;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -47,15 +55,48 @@ public abstract class AbstractLensDriver implements LensDriver {
@Getter
private String fullyQualifiedName = null;
- private DriverQueryHook noOpDriverQueryHook = new NoOpDriverQueryHook();
+ @Getter
+ private Configuration conf;
+
+ @Getter
+ private ImmutableSet<QueryLaunchingConstraint> queryConstraints;
+ @Getter
+ private ImmutableSet<WaitingQueriesSelectionPolicy> waitingQuerySelectionPolicies;
+ @Getter
+ RetryPolicyDecider<QueryContext> retryPolicyDecider;
+ @Getter
+ private DriverQueryHook queryHook;
@Override
public void configure(Configuration conf, String driverType, String driverName) throws LensException {
if (StringUtils.isBlank(driverType) || StringUtils.isBlank(driverName)) {
throw new LensException("Driver Type and Name can not be null or empty");
}
- fullyQualifiedName = new StringBuilder(driverType).append(SEPARATOR).append(driverName).toString();
- noOpDriverQueryHook.setDriver(this);
+ fullyQualifiedName = driverType + SEPARATOR + driverName;
+ this.conf = new DriverConfiguration(conf, driverType, getClass());
+ this.conf.addResource(getClass().getSimpleName().toLowerCase() + "-default.xml");
+ this.conf.addResource(getDriverResourcePath(getClass().getSimpleName().toLowerCase() + "-site.xml"));
+
+ this.queryConstraints = getImplementations(QUERY_LAUNCHING_CONSTRAINT_FACTORIES_SFX, getConf());
+ this.waitingQuerySelectionPolicies = getImplementations(WAITING_QUERIES_SELECTION_POLICY_FACTORIES_SFX, getConf());
+
+ loadRetryPolicyDecider();
+ loadQueryHook();
+ }
+
+ protected void loadQueryHook() throws LensException {
+ try {
+ queryHook = getConf().getClass(
+ DRIVER_HOOK_CLASS_SFX, NoOpDriverQueryHook.class, DriverQueryHook.class
+ ).newInstance();
+ queryHook.setDriver(this);
+ } catch (InstantiationException | IllegalAccessException e) {
+ throw new LensException("Can't instantiate driver query hook for hivedriver with given class", e);
+ }
+ }
+
+ protected void loadRetryPolicyDecider() throws LensException {
+ this.retryPolicyDecider = ChainedRetryPolicyDecider.from(getConf(), RETRY_POLICY_CLASSES_SFX);
}
/**
@@ -96,8 +137,8 @@ public abstract class AbstractLensDriver implements LensDriver {
* @return
*/
protected String getDriverResourcePath(String resourceName) {
- return new StringBuilder(LensConfConstants.DRIVERS_BASE_DIR).append(SEPARATOR).append(getFullyQualifiedName())
- .append(SEPARATOR).append(resourceName).toString();
+ return LensConfConstants.DRIVERS_BASE_DIR + SEPARATOR + getFullyQualifiedName()
+ + SEPARATOR + resourceName;
}
@Override
@@ -106,11 +147,6 @@ public abstract class AbstractLensDriver implements LensDriver {
}
@Override
- public DriverQueryHook getQueryHook() {
- return noOpDriverQueryHook;
- }
-
- @Override
public StatusUpdateMethod getStatusUpdateMethod() {
return StatusUpdateMethod.PULL;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
new file mode 100644
index 0000000..69a1a0b
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverConfiguration.java
@@ -0,0 +1,60 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.driver;
+
+
+import static org.apache.lens.server.api.LensConfConstants.DRIVER_PFX;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class DriverConfiguration extends Configuration {
+ private final String driverClassType;
+ private String driverType;
+ private final Class<? extends AbstractLensDriver> driverClass;
+
+ public DriverConfiguration(Configuration conf, String driverType, Class<? extends AbstractLensDriver> driverClass) {
+ super(conf);
+ this.driverType = driverType;
+ this.driverClass = driverClass;
+ this.driverClassType = driverClass.getSimpleName().toLowerCase().replaceAll("driver$", "");
+ }
+
+ @Override
+ public String[] getStrings(String name) {
+ for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name,
+ DRIVER_PFX + name, name, }) {
+ String[] s = super.getStrings(key);
+ if (s != null) {
+ return s;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public <U> Class<? extends U> getClass(String name, Class<? extends U> defaultValue, Class<U> xface) {
+ for (String key : new String[]{DRIVER_PFX + driverType + "." + name, DRIVER_PFX + driverClassType + "." + name,
+ DRIVER_PFX + name, name, }) {
+ if (getTrimmed(key) != null) {
+ return super.getClass(key, defaultValue, xface);
+ }
+ }
+ return defaultValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
index fc24fc6..b5c5dcd 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/DriverQueryStatus.java
@@ -22,12 +22,12 @@ import java.io.Serializable;
import org.apache.lens.api.query.QueryStatus;
-import lombok.Getter;
-import lombok.Setter;
+import lombok.Data;
/**
* The Class DriverQueryStatus.
*/
+@Data
public class DriverQueryStatus implements Serializable {
/**
@@ -35,6 +35,9 @@ public class DriverQueryStatus implements Serializable {
*/
private static final long serialVersionUID = 1L;
+ public boolean failed() {
+ return state == DriverQueryState.FAILED;
+ }
/**
@@ -96,59 +99,56 @@ public class DriverQueryStatus implements Serializable {
/**
* The progress.
*/
- @Getter
- @Setter
- private double progress = 0.0f;
+ private double progress;
/**
* The state.
*/
- @Getter
- @Setter
- private DriverQueryState state = DriverQueryState.NEW;
+ private DriverQueryState state;
/**
* The status message.
*/
- @Getter
- @Setter
private String statusMessage;
/**
* The is result set available.
*/
- @Getter
- @Setter
- private boolean isResultSetAvailable = false;
+ private boolean isResultSetAvailable;
/**
* The progress message.
*/
- @Getter
- @Setter
private String progressMessage;
/**
* The error message.
*/
- @Getter
- @Setter
private String errorMessage;
/**
* The driver start time.
*/
- @Getter
- @Setter
- private Long driverStartTime = 0L;
+ private Long driverStartTime;
/**
* The driver finish time.
*/
- @Getter
- @Setter
- private Long driverFinishTime = 0L;
+ private Long driverFinishTime;
+ {
+ clear();
+ }
+ public void clear() {
+ progress = 0.0f;
+ state = DriverQueryState.NEW;
+ statusMessage = null;
+ isResultSetAvailable = false;
+ progressMessage = null;
+ errorMessage = null;
+ driverStartTime = 0L;
+ driverFinishTime = 0L;
+ }
/**
* To query status.
*
@@ -169,7 +169,7 @@ public class DriverQueryStatus implements Serializable {
qstate = QueryStatus.Status.EXECUTED;
break;
case FAILED:
- qstate = QueryStatus.Status.FAILED;
+ qstate = QueryStatus.Status.FAILING;
break;
case CANCELED:
qstate = QueryStatus.Status.CANCELED;
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
index e472de0..1462239 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensDriver.java
@@ -31,6 +31,7 @@ import org.apache.lens.server.api.query.QueryContext;
import org.apache.lens.server.api.query.collect.WaitingQueriesSelectionPolicy;
import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
import org.apache.lens.server.api.query.cost.QueryCost;
+import org.apache.lens.server.api.retry.RetryPolicyDecider;
import org.apache.hadoop.conf.Configuration;
@@ -232,4 +233,6 @@ public interface LensDriver extends Externalizable {
* @return The method of status update supported by this driver.
*/
StatusUpdateMethod getStatusUpdateMethod();
+
+ RetryPolicyDecider<QueryContext> getRetryPolicyDecider();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
index e160f58..c6a872d 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/AbstractQueryContext.java
@@ -202,7 +202,7 @@ public abstract class AbstractQueryContext implements Serializable {
public Map<LensDriver, DriverEstimateRunnable> getDriverEstimateRunnables() throws LensException {
Map<LensDriver, DriverEstimateRunnable> estimateRunnables = new HashMap<LensDriver, DriverEstimateRunnable>();
- for (LensDriver driver : driverContext.getDrivers()) {
+ for (LensDriver driver : driverContext.getEligibleDrivers()) {
estimateRunnables.put(driver, new DriverEstimateRunnable(this, driver));
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 5ff59bd..8e431d1 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -50,8 +50,8 @@ public class DriverSelectorQueryContext {
*/
@Getter
@Setter
- protected Map<LensDriver, DriverQueryContext> driverQueryContextMap = new HashMap<LensDriver,
- DriverQueryContext>();
+ protected Map<LensDriver, DriverQueryContext> driverQueryContextMap = new HashMap<>();
+ private Set<LensDriver> blackListedDrivers = Sets.newHashSet();
public DriverSelectorQueryContext(final String userQuery, final Configuration queryConf,
final Collection<LensDriver> drivers) {
@@ -77,6 +77,10 @@ public class DriverSelectorQueryContext {
}
}
+ public void blacklist(LensDriver selectedDriver) {
+ blackListedDrivers.add(selectedDriver);
+ }
+
public static class DriverQueryContext {
@Getter
@@ -264,13 +268,16 @@ public class DriverSelectorQueryContext {
public Collection<LensDriver> getDrivers() {
return driverQueryContextMap.keySet();
}
+ public Collection<LensDriver> getEligibleDrivers() {
+ return Sets.difference(driverQueryContextMap.keySet(), blackListedDrivers);
+ }
public Collection<LensDriver> getDriversWithValidQueryCost() {
final Set<LensDriver> eligibleDrivers = Sets.newLinkedHashSet();
- for (Map.Entry<LensDriver, DriverQueryContext> driverToDriverContext : this.driverQueryContextMap.entrySet()) {
- if (driverToDriverContext.getValue().driverCost != null) {
- eligibleDrivers.add(driverToDriverContext.getKey());
+ for (LensDriver driver: getEligibleDrivers()) {
+ if (driverQueryContextMap.get(driver).driverCost != null) {
+ eligibleDrivers.add(driver);
}
}
return Collections.unmodifiableCollection(eligibleDrivers);
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index b58fcf9..d88944b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -19,10 +19,11 @@
package org.apache.lens.server.api.query;
import java.util.Collection;
-import java.util.Iterator;
+import java.util.List;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.Priority;
+import org.apache.lens.api.query.FailedAttempt;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.driver.LensDriver;
@@ -46,7 +47,7 @@ import lombok.ToString;
*
* @see java.lang.Object#hashCode()
*/
-@EqualsAndHashCode(exclude = {"selectedDriver", "conf"})
+@EqualsAndHashCode(exclude = {"selectedDriver", "conf", "failedAttempts"})
/*
* (non-Javadoc)
*
@@ -185,6 +186,10 @@ public class FinishedLensQuery {
@Setter
private String driverQuery;
+ @Getter
+ @Setter
+ private List<FailedAttempt> failedAttempts;
+
/**
* Instantiates a new finished lens query.
*/
@@ -222,6 +227,7 @@ public class FinishedLensQuery {
this.priority = ctx.getPriority().toString();
}
this.conf = ctx.getLensConf();
+ this.failedAttempts = ctx.getFailedAttempts();
}
public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) {
@@ -249,13 +255,12 @@ public class FinishedLensQuery {
if (getPriority() != null) {
qctx.setPriority(Priority.valueOf(getPriority()));
}
+ qctx.setFailedAttempts(getFailedAttempts());
return qctx;
}
private LensDriver getDriverFromName(Collection<LensDriver> drivers) {
- Iterator<LensDriver> iterator = drivers.iterator();
- while (iterator.hasNext()) {
- LensDriver driver = iterator.next();
+ for (LensDriver driver : drivers) {
if (driverName.equals(driver.getFullyQualifiedName())) {
return driver;
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java
deleted file mode 100644
index eaf3fee..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/PriorityChange.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.Priority;
-import org.apache.lens.api.query.QueryHandle;
-
-/**
- * Event fired when query priority changes.
- */
-public class PriorityChange extends QueryEvent<Priority> {
-
- /**
- * Instantiates a new priority change.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public PriorityChange(long eventTime, Priority prev, Priority current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java
deleted file mode 100644
index 8980b61..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryAccepted.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-
-/**
- * The Class QueryAccepted.
- */
-public class QueryAccepted extends QueryEvent<String> {
-
- /**
- * Instantiates a new query accepted.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueryAccepted(long eventTime, String prev, String current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java
deleted file mode 100644
index a473a47..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryCancelled.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when query is cancelled.
- */
-public class QueryCancelled extends QueryEnded {
-
- /**
- * Instantiates a new query cancelled.
- *
- * @param ctx the query context
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- * @param user the user
- * @param cause the cause
- */
- public QueryCancelled(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
- QueryHandle handle,
- String user, String cause) {
- super(ctx, eventTime, prev, current, handle, user, cause);
- checkCurrentState(QueryStatus.Status.CANCELED);
- }
-
- public QueryCancelled(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
- // TODO: correct username. put who cancelled it, not the submitter. Similar for others
- this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java
deleted file mode 100644
index 3837087..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryClosed.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when a query is closed.
- */
-public class QueryClosed extends QueryEnded {
-
- /**
- * Instantiates a new query closed.
- *
- * @param ctx the query context
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- * @param user the user
- * @param cause the cause
- */
- public QueryClosed(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
- QueryHandle handle,
- String user, String cause) {
- super(ctx, eventTime, prev, current, handle, user, cause);
- checkCurrentState(QueryStatus.Status.CLOSED);
- }
-
- public QueryClosed(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
- this(ctx, ctx.getClosedTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
- }
-}
[3/4] lens git commit: LENS-743: Query retry framework for retrying
upon transient failures
Posted by pr...@apache.org.
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index d0662f4..63d3539 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -27,16 +27,17 @@ import java.util.*;
import java.util.concurrent.Future;
import org.apache.lens.api.LensConf;
+import org.apache.lens.api.query.FailedAttempt;
import org.apache.lens.api.query.LensQuery;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.query.QueryStatus.Status;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.common.BackOffRetryHandler;
-import org.apache.lens.server.api.common.FailureContext;
import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.constraint.QueryLaunchingConstraint;
+import org.apache.lens.server.api.retry.BackOffRetryHandler;
+import org.apache.lens.server.api.retry.FailureContext;
import org.apache.lens.server.api.util.LensUtil;
import org.apache.hadoop.conf.Configuration;
@@ -53,7 +54,7 @@ import lombok.extern.slf4j.Slf4j;
* The Class QueryContext.
*/
@Slf4j
-public class QueryContext extends AbstractQueryContext {
+public class QueryContext extends AbstractQueryContext implements FailureContext {
/**
* The Constant serialVersionUID.
@@ -191,7 +192,7 @@ public class QueryContext extends AbstractQueryContext {
@Setter
private byte[] queryConfHash;
- transient FailureContext statusUpdateFailures = new FailureContext();
+ transient StatusUpdateFailureContext statusUpdateFailures = new StatusUpdateFailureContext();
@Getter
@Setter
@@ -200,7 +201,18 @@ public class QueryContext extends AbstractQueryContext {
@Getter
@Setter
private transient Future queryLauncher;
+
private final List<QueryDriverStatusUpdateListener> driverStatusUpdateListeners = Lists.newArrayList();
+ @Getter
+ @Setter
+ List<FailedAttempt> failedAttempts = Lists.newArrayList();
+
+ @Getter
+ @Setter
+ private BackOffRetryHandler<QueryContext> driverRetryPolicy;
+ @Getter
+ @Setter
+ private BackOffRetryHandler<QueryContext> serverRetryPolicy;
/**
* Creates context from query
@@ -224,7 +236,7 @@ public class QueryContext extends AbstractQueryContext {
*/
public QueryContext(PreparedQueryContext prepared, String user, LensConf qconf, Configuration conf) {
this(prepared.getUserQuery(), user, qconf, mergeConf(prepared.getConf(), conf), prepared.getDriverContext()
- .getDriverQueryContextMap().keySet(), prepared.getDriverContext().getSelectedDriver(), true);
+ .getDriverQueryContextMap().keySet(), prepared.getDriverContext().getSelectedDriver(), true);
setDriverContext(prepared.getDriverContext());
setSelectedDriverQuery(prepared.getSelectedDriverQuery());
setSelectedDriverQueryCost(prepared.getSelectedDriverQueryCost());
@@ -241,7 +253,7 @@ public class QueryContext extends AbstractQueryContext {
* @param selectedDriver SelectedDriver
*/
QueryContext(String userQuery, String user, LensConf qconf, Configuration conf, Collection<LensDriver> drivers,
- LensDriver selectedDriver, boolean mergeDriverConf) {
+ LensDriver selectedDriver, boolean mergeDriverConf) {
this(userQuery, user, qconf, conf, drivers, selectedDriver, System.currentTimeMillis(), mergeDriverConf);
}
@@ -257,7 +269,7 @@ public class QueryContext extends AbstractQueryContext {
* @param submissionTime the submission time
*/
QueryContext(String userQuery, String user, LensConf qconf, Configuration conf, Collection<LensDriver> drivers,
- LensDriver selectedDriver, long submissionTime, boolean mergeDriverConf) {
+ LensDriver selectedDriver, long submissionTime, boolean mergeDriverConf) {
super(userQuery, user, qconf, conf, drivers, mergeDriverConf);
this.submissionTime = submissionTime;
this.queryHandle = new QueryHandle(UUID.randomUUID());
@@ -265,9 +277,9 @@ public class QueryContext extends AbstractQueryContext {
this.lensConf = qconf;
this.conf = conf;
this.isPersistent = conf.getBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_SET,
- LensConfConstants.DEFAULT_PERSISTENT_RESULT_SET);
+ LensConfConstants.DEFAULT_PERSISTENT_RESULT_SET);
this.isDriverPersistent = conf.getBoolean(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER,
- LensConfConstants.DEFAULT_DRIVER_PERSISTENT_RESULT_SET);
+ LensConfConstants.DEFAULT_DRIVER_PERSISTENT_RESULT_SET);
this.userQuery = userQuery;
if (selectedDriver != null) {
this.setSelectedDriver(selectedDriver);
@@ -289,7 +301,7 @@ public class QueryContext extends AbstractQueryContext {
* @return QueryContext object
*/
public static QueryContext createContextWithSingleDriver(String query, String user, LensConf qconf,
- Configuration conf, LensDriver driver, String lensSessionPublicId, boolean mergeDriverConf) {
+ Configuration conf, LensDriver driver, String lensSessionPublicId, boolean mergeDriverConf) {
QueryContext ctx = new QueryContext(query, user, qconf, conf, Lists.newArrayList(driver), driver, mergeDriverConf);
ctx.setLensSessionIdentifier(lensSessionPublicId);
return ctx;
@@ -297,7 +309,7 @@ public class QueryContext extends AbstractQueryContext {
public void initTransientState() {
super.initTransientState();
- statusUpdateFailures = new FailureContext();
+ statusUpdateFailures = new StatusUpdateFailureContext();
}
/**
@@ -338,7 +350,7 @@ public class QueryContext extends AbstractQueryContext {
getSelectedDriverQuery(),
status,
resultSetPath, driverOpHandle, lensConf, submissionTime, launchTime, driverStatus.getDriverStartTime(),
- driverStatus.getDriverFinishTime(), endTime, closedTime, queryName);
+ driverStatus.getDriverFinishTime(), endTime, closedTime, queryName, getFailedAttempts());
}
public boolean isResultAvailableInDriver() {
@@ -505,8 +517,8 @@ public class QueryContext extends AbstractQueryContext {
* 5. rowsToPreFetch should be > 0
*/
if (isPersistent && executeTimeoutMillis > 0
- && result instanceof InMemoryResultSet
- && conf.getBoolean(PREFETCH_INMEMORY_RESULTSET, DEFAULT_PREFETCH_INMEMORY_RESULTSET)) {
+ && result instanceof InMemoryResultSet
+ && conf.getBoolean(PREFETCH_INMEMORY_RESULTSET, DEFAULT_PREFETCH_INMEMORY_RESULTSET)) {
int rowsToPreFetch = conf.getInt(PREFETCH_INMEMORY_RESULTSET_ROWS, DEFAULT_PREFETCH_INMEMORY_RESULTSET_ROWS);
if (rowsToPreFetch > 0) {
long executeTimeOutTime = submissionTime + executeTimeoutMillis;
@@ -579,4 +591,32 @@ public class QueryContext extends AbstractQueryContext {
this.driverStatusUpdateListeners.add(driverStatusUpdateListener);
}
}
+
+ @Override
+ public long getLastFailedTime() {
+ if (getFailCount() == 0) {
+ return 0;
+ }
+ return getFailedAttempts().get(getFailedAttempts().size() - 1).getDriverFinishTime();
+ }
+
+ @Override
+ public int getFailCount() {
+ return getFailedAttempts().size();
+ }
+
+ public BackOffRetryHandler<QueryContext> getRetryPolicy() {
+ return driverRetryPolicy != null ? driverRetryPolicy : serverRetryPolicy;
+ }
+
+ public void extractFailedAttempt() {
+ extractFailedAttempt(getSelectedDriver());
+ }
+
+ public void extractFailedAttempt(LensDriver selectedDriver) {
+ failedAttempts.add(new FailedAttempt(selectedDriver.getFullyQualifiedName(), getDriverStatus().getProgress(),
+ getDriverStatus().getProgressMessage(), getDriverStatus().getErrorMessage(),
+ getDriverStatus().getDriverStartTime(), getDriverStatus().getDriverFinishTime()));
+ getDriverStatus().clear();
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java
deleted file mode 100644
index e80da6d..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEnded.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import java.util.EnumSet;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-import org.apache.commons.lang.StringUtils;
-
-import lombok.Getter;
-
-/**
- * Generic event denoting that query has ended. If a listener wants to just be notified when query has ended
- * irrespective of its success or failure, then that listener can subscribe for this event type
- */
-public class QueryEnded extends StatusChange {
-
- @Getter
- private final QueryContext queryContext;
- /**
- * The user.
- */
- @Getter
- private final String user;
-
- /**
- * The cause.
- */
- @Getter
- private final String cause;
-
- /**
- * The Constant END_STATES.
- */
- public static final EnumSet<QueryStatus.Status> END_STATES = EnumSet.of(QueryStatus.Status.SUCCESSFUL,
- QueryStatus.Status.CANCELED, QueryStatus.Status.CLOSED, QueryStatus.Status.FAILED);
-
- /**
- * Instantiates a new query ended.
- *
- * @param ctx
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- * @param user the user
- * @param cause the cause
- */
- public QueryEnded(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
- QueryHandle handle, String user, String cause) {
- super(eventTime, prev, current, handle);
- this.queryContext = ctx;
- this.user = user;
- this.cause = cause;
- if (!END_STATES.contains(current)) {
- throw new IllegalStateException("Not a valid end state: " + current + " query: " + handle);
- }
- }
-
- public String toString() {
- StringBuilder buf = new StringBuilder(super.toString());
- if (StringUtils.isNotBlank(cause)) {
- buf.append(" cause:").append(cause);
- }
- return buf.toString();
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java
deleted file mode 100644
index 81c53be..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryEvent.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import java.util.UUID;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.server.api.events.LensEvent;
-
-import lombok.Getter;
-
-/**
- * A generic event related to state change of a query Subclasses must declare the specific type of change they are
- * interested in.
- * <p></p>
- * Every event will have an ID, which should be used by listeners to check if the event is already received.
- *
- * @param <T> Type of changed information about the query
- */
-public abstract class QueryEvent<T> extends LensEvent {
-
- /**
- * The previous value.
- */
- @Getter
- protected final T previousValue;
-
- /**
- * The current value.
- */
- @Getter
- protected final T currentValue;
-
- /**
- * The query handle.
- */
- @Getter
- protected final QueryHandle queryHandle;
-
- /**
- * The id.
- */
- protected final UUID id = UUID.randomUUID();
-
- /**
- * Instantiates a new query event.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueryEvent(long eventTime, T prev, T current, QueryHandle handle) {
- super(eventTime);
- previousValue = prev;
- currentValue = current;
- this.queryHandle = handle;
- }
-
- @Override
- public String getEventId() {
- return id.toString();
- }
-
- /*
- * (non-Javadoc)
- *
- * @see java.lang.Object#toString()
- */
- @Override
- public String toString() {
- StringBuilder buf = new StringBuilder("QueryEvent: ").append(getClass().getSimpleName()).append(":{id: ")
- .append(id).append(", query:").append(getQueryHandle()).append(", change:[").append(previousValue)
- .append(" -> ").append(currentValue).append("]}");
- return buf.toString();
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecuted.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecuted.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecuted.java
deleted file mode 100644
index af8c8ee..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryExecuted.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when query is successfully completed by the driver.
- */
-public class QueryExecuted extends StatusChange {
-
- /**
- * Instantiates a new query executed.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueryExecuted(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- checkCurrentState(QueryStatus.Status.EXECUTED);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryFailed.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryFailed.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryFailed.java
deleted file mode 100644
index bdffbc2..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryFailed.java
+++ /dev/null
@@ -1,50 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when a query fails to execute. Use getCause() to get the reason for failure.
- */
-public class QueryFailed extends QueryEnded {
-
- /**
- * Instantiates a new query failed.
- *
- * @param ctx the query context
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- * @param user the user
- * @param cause the cause
- */
- public QueryFailed(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
- QueryHandle handle,
- String user, String cause) {
- super(ctx, eventTime, prev, current, handle, user, cause);
- checkCurrentState(QueryStatus.Status.FAILED);
- }
-
- public QueryFailed(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
- this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryLaunched.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryLaunched.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryLaunched.java
deleted file mode 100644
index 5fcdd73..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryLaunched.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when query is LAUNCHED.
- */
-public class QueryLaunched extends StatusChange {
-
- /**
- * Instantiates a new query launched.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueryLaunched(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- checkCurrentState(QueryStatus.Status.LAUNCHED);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryQueued.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryQueued.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryQueued.java
deleted file mode 100644
index 5f347ad..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryQueued.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when a query is QUEUED.
- */
-public class QueryQueued extends StatusChange {
-
- /**
- * The user.
- */
- private final String user;
-
- /**
- * Instantiates a new query queued.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- * @param user the user
- */
- public QueryQueued(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle,
- String user) {
- super(eventTime, prev, current, handle);
- checkCurrentState(QueryStatus.Status.QUEUED);
- this.user = user;
- }
-
- /**
- * Get the submitting user
- *
- * @return user
- */
- public final String getUser() {
- return user;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRejected.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRejected.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRejected.java
deleted file mode 100644
index c4e60e3..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRejected.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-
-/**
- * The Class QueryRejected.
- */
-public class QueryRejected extends QueryEvent<String> {
-
- /**
- * Instantiates a new query rejected.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueryRejected(long eventTime, String prev, String current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRunning.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRunning.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRunning.java
deleted file mode 100644
index 52aa50d..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryRunning.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when query enters a RUNNING state.
- */
-public class QueryRunning extends StatusChange {
-
- /**
- * Instantiates a new query running.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueryRunning(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- checkCurrentState(QueryStatus.Status.RUNNING);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QuerySuccess.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QuerySuccess.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QuerySuccess.java
deleted file mode 100644
index 298fdbb..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QuerySuccess.java
+++ /dev/null
@@ -1,47 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * Event fired when query is successfully completed.
- */
-public class QuerySuccess extends QueryEnded {
-
- /**
- * Instantiates a new query success.
- *
- * @param ctx the query context
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QuerySuccess(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
- QueryHandle handle) {
- super(ctx, eventTime, prev, current, handle, null, null);
- checkCurrentState(QueryStatus.Status.SUCCESSFUL);
- }
-
- public QuerySuccess(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState) {
- this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle());
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueuePositionChange.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueuePositionChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueuePositionChange.java
deleted file mode 100644
index 062e14e..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueuePositionChange.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-
-/**
- * Event fired when query moves up or down in the execution engine's queue.
- */
-public class QueuePositionChange extends QueryEvent<Integer> {
-
- /**
- * Instantiates a new queue position change.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public QueuePositionChange(long eventTime, Integer prev, Integer current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- }
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusChange.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusChange.java
deleted file mode 100644
index 949ec20..0000000
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusChange.java
+++ /dev/null
@@ -1,52 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.lens.server.api.query;
-
-import org.apache.lens.api.query.QueryHandle;
-import org.apache.lens.api.query.QueryStatus;
-
-/**
- * The Class StatusChange.
- */
-public abstract class StatusChange extends QueryEvent<QueryStatus.Status> {
-
- /**
- * Instantiates a new status change.
- *
- * @param eventTime the event time
- * @param prev the prev
- * @param current the current
- * @param handle the handle
- */
- public StatusChange(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
- super(eventTime, prev, current, handle);
- }
-
- /**
- * Check current state.
- *
- * @param status the status
- */
- protected void checkCurrentState(QueryStatus.Status status) {
- if (currentValue != status) {
- throw new IllegalStateException("Invalid query state: " + currentValue + " query:" + queryHandle.toString());
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusUpdateFailureContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusUpdateFailureContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusUpdateFailureContext.java
new file mode 100644
index 0000000..13ecd43
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/StatusUpdateFailureContext.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.api.query;
+
+import org.apache.lens.server.api.retry.FailureContext;
+
+import lombok.Getter;
+
+/**
+ * Failure context captures last failure time and number of failures.
+ */
+public class StatusUpdateFailureContext implements FailureContext {
+
+ @Getter
+ private long lastFailedTime = 0;
+ @Getter
+ private int failCount = 0;
+
+ public synchronized void updateFailure() {
+ lastFailedTime = System.currentTimeMillis();
+ failCount++;
+ }
+
+ public synchronized void clear() {
+ lastFailedTime = 0;
+ failCount = 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/ChainedComparator.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/ChainedComparator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/ChainedComparator.java
new file mode 100644
index 0000000..2cff8d8
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/ChainedComparator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.comparators;
+
+
+import java.util.Comparator;
+import java.util.List;
+
+import lombok.Data;
+
+@Data
+public class ChainedComparator<T> implements Comparator<T> {
+ private final List<Comparator<T>> comparators;
+
+ @Override
+ public int compare(T o1, T o2) {
+ for (Comparator<T> comparator : comparators) {
+ int cmp = comparator.compare(o1, o2);
+ if (cmp != 0) {
+ return cmp;
+ }
+ }
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/FIFOQueryComparator.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/FIFOQueryComparator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/FIFOQueryComparator.java
new file mode 100644
index 0000000..3596729
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/FIFOQueryComparator.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.comparators;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+public class FIFOQueryComparator implements QueryComparator {
+
+ @Override
+ public int compare(QueryContext o1, QueryContext o2) {
+ return Long.compare(o1.getSubmissionTime(), o2.getSubmissionTime());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/MoreRetriesFirstComparator.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/MoreRetriesFirstComparator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/MoreRetriesFirstComparator.java
new file mode 100644
index 0000000..d3242bd
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/MoreRetriesFirstComparator.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.comparators;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+public class MoreRetriesFirstComparator implements QueryComparator {
+
+ @Override
+ public int compare(final QueryContext o1, final QueryContext o2) {
+ // swap order for reverse sorting
+ return Integer.compare(o2.getFailedAttempts().size(), o1.getFailedAttempts().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryComparator.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryComparator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryComparator.java
new file mode 100644
index 0000000..a633960
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryComparator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+package org.apache.lens.server.api.query.comparators;
+
+import java.util.Comparator;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+public interface QueryComparator extends Comparator<QueryContext> {
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryCostComparator.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryCostComparator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryCostComparator.java
new file mode 100644
index 0000000..193c277
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryCostComparator.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.comparators;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+public class QueryCostComparator implements QueryComparator {
+
+ @Override
+ public int compare(final QueryContext o1, final QueryContext o2) {
+ return o1.getSelectedDriverQueryCost().compareTo(o2.getSelectedDriverQueryCost());
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryPriorityComparator.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryPriorityComparator.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryPriorityComparator.java
new file mode 100644
index 0000000..7ef1a9d
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/comparators/QueryPriorityComparator.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.comparators;
+
+import org.apache.lens.server.api.query.QueryContext;
+
+public class QueryPriorityComparator implements QueryComparator {
+
+ @Override
+ public int compare(final QueryContext o1, final QueryContext o2) {
+ return o1.getPriority().compareTo(o2.getPriority());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
index e0f1376..8314977 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/MaxConcurrentDriverQueriesConstraint.java
@@ -41,22 +41,50 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst
private final int maxConcurrentLaunches;
@Override
- public boolean allowsLaunchOf(
+ public String allowsLaunchOf(
final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries) {
final LensDriver selectedDriver = candidateQuery.getSelectedDriver();
final Set<QueryContext> driverLaunchedQueries = launchedQueries.getQueries(selectedDriver);
- final boolean canLaunch = (launchedQueries.getQueriesCount(selectedDriver) < maxConcurrentQueries)
- && (getIsLaunchingCount(driverLaunchedQueries) < maxConcurrentLaunches)
- && canLaunchWithQueueConstraint(candidateQuery, driverLaunchedQueries)
- && canLaunchWithPriorityConstraint(candidateQuery, driverLaunchedQueries);
- log.debug("canLaunch:{}", canLaunch);
- return canLaunch;
+
+ String maxConcurrentLimitation = canLaunchWithMaxConcurrentConstraint(candidateQuery,
+ launchedQueries.getQueriesCount(selectedDriver));
+ if (maxConcurrentLimitation != null) {
+ return maxConcurrentLimitation;
+ }
+ String maxLaunchingLimitation = canLaunchWithMaxLaunchingConstraint(driverLaunchedQueries);
+ if (maxLaunchingLimitation != null) {
+ return maxLaunchingLimitation;
+ }
+ String queueLimitation = canLaunchWithQueueConstraint(candidateQuery, driverLaunchedQueries);
+ if (queueLimitation != null) {
+ return queueLimitation;
+ }
+ String priorityLimitation = canLaunchWithPriorityConstraint(candidateQuery, driverLaunchedQueries);
+ if (priorityLimitation != null) {
+ return priorityLimitation;
+ }
+ return null;
+ }
+
+ private String canLaunchWithMaxLaunchingConstraint(Set<QueryContext> driverLaunchedQueries) {
+ int launchingCount = getIsLaunchingCount(driverLaunchedQueries);
+ if (launchingCount >= maxConcurrentLaunches) {
+ return launchingCount + "/" + maxConcurrentLaunches + " launches happening";
+ }
+ return null;
}
- private boolean canLaunchWithQueueConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) {
+ private String canLaunchWithMaxConcurrentConstraint(QueryContext candidateQuery, int concurrentLaunched) {
+ if (concurrentLaunched >= maxConcurrentQueries) {
+ return concurrentLaunched + "/" + maxConcurrentQueries + " queries running on "
+ + candidateQuery.getSelectedDriver().getFullyQualifiedName();
+ }
+ return null;
+ }
+ private String canLaunchWithQueueConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) {
if (maxConcurrentQueriesPerQueue == null) {
- return true;
+ return null;
}
String queue = candidateQuery.getQueue();
Integer limit = maxConcurrentQueriesPerQueue.get(queue);
@@ -64,7 +92,7 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst
if (defaultMaxConcurrentQueriesPerQueueLimit != null) { //Check if any default limit is enabled for all queues
limit = defaultMaxConcurrentQueriesPerQueueLimit;
} else {
- return true;
+ return null;
}
}
int launchedOnQueue = 0;
@@ -73,17 +101,20 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst
launchedOnQueue++;
}
}
- return launchedOnQueue < limit;
+ if (launchedOnQueue >= limit) {
+ return launchedOnQueue + "/" + limit + " queries running in Queue " + queue;
+ }
+ return null;
}
- private boolean canLaunchWithPriorityConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) {
+ private String canLaunchWithPriorityConstraint(QueryContext candidateQuery, Set<QueryContext> launchedQueries) {
if (maxConcurrentQueriesPerPriority == null) {
- return true;
+ return null;
}
Priority priority = candidateQuery.getPriority();
Integer limit = maxConcurrentQueriesPerPriority.get(priority);
if (limit == null) {
- return true;
+ return null;
}
int launchedOnPriority = 0;
for (QueryContext context : launchedQueries) {
@@ -91,7 +122,10 @@ public class MaxConcurrentDriverQueriesConstraint implements QueryLaunchingConst
launchedOnPriority++;
}
}
- return launchedOnPriority < limit;
+ if (launchedOnPriority >= limit) {
+ return launchedOnPriority + "/" + limit + " queries running with priority " + priority;
+ }
+ return null;
}
private int getIsLaunchingCount(final Set<QueryContext> launchedQueries) {
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java
index 12d9562..29142eb 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/constraint/QueryLaunchingConstraint.java
@@ -28,7 +28,7 @@ public interface QueryLaunchingConstraint {
*
* @param candidateQuery The query which is the next candidate to be launched.
* @param launchedQueries Current launched queries
- * @return
+ * @return null if allowed to launch, otherwise a String containing the reason to block launch
*/
- boolean allowsLaunchOf(final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries);
+ String allowsLaunchOf(final QueryContext candidateQuery, final EstimatedImmutableQueryCollection launchedQueries);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/PriorityChange.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/PriorityChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/PriorityChange.java
new file mode 100644
index 0000000..72d5eb7
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/PriorityChange.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.Priority;
+import org.apache.lens.api.query.QueryHandle;
+
+/**
+ * Event fired when query priority changes.
+ */
+public class PriorityChange extends QueryEvent<Priority> {
+
+ /**
+ * Instantiates a new priority change.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public PriorityChange(long eventTime, Priority prev, Priority current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryAccepted.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryAccepted.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryAccepted.java
new file mode 100644
index 0000000..5ab7cb4
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryAccepted.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+
+/**
+ * The Class QueryAccepted.
+ */
+public class QueryAccepted extends QueryEvent<String> {
+
+ /**
+ * Instantiates a new query accepted.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QueryAccepted(long eventTime, String prev, String current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryCancelled.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryCancelled.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryCancelled.java
new file mode 100644
index 0000000..9198e02
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryCancelled.java
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ * Event fired when query is cancelled.
+ */
+public class QueryCancelled extends QueryEnded {
+
+ /**
+ * Instantiates a new query cancelled.
+ *
+ * @param ctx the query context
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ * @param user the user
+ * @param cause the cause
+ */
+ public QueryCancelled(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
+ QueryHandle handle,
+ String user, String cause) {
+ super(ctx, eventTime, prev, current, handle, user, cause);
+ checkCurrentState(QueryStatus.Status.CANCELED);
+ }
+
+ public QueryCancelled(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
+ // TODO: correct username. put who cancelled it, not the submitter. Similar for others
+ this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryClosed.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryClosed.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryClosed.java
new file mode 100644
index 0000000..289159e
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryClosed.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ * Event fired when a query is closed.
+ */
+public class QueryClosed extends QueryEnded {
+
+ /**
+ * Instantiates a new query closed.
+ *
+ * @param ctx the query context
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ * @param user the user
+ * @param cause the cause
+ */
+ public QueryClosed(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
+ QueryHandle handle,
+ String user, String cause) {
+ super(ctx, eventTime, prev, current, handle, user, cause);
+ checkCurrentState(QueryStatus.Status.CLOSED);
+ }
+
+ public QueryClosed(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
+ this(ctx, ctx.getClosedTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEnded.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEnded.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEnded.java
new file mode 100644
index 0000000..cf052ef
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEnded.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import java.util.EnumSet;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.server.api.query.QueryContext;
+
+import org.apache.commons.lang.StringUtils;
+
+import lombok.Getter;
+
+/**
+ * Generic event denoting that query has ended. If a listener wants to just be notified when query has ended
+ * irrespective of its success or failure, then that listener can subscribe for this event type
+ */
+public class QueryEnded extends StatusChange {
+
+ @Getter
+ private final QueryContext queryContext;
+ /**
+ * The user.
+ */
+ @Getter
+ private final String user;
+
+ /**
+ * The cause.
+ */
+ @Getter
+ private final String cause;
+
+ /**
+ * The Constant END_STATES.
+ */
+ public static final EnumSet<QueryStatus.Status> END_STATES = EnumSet.of(QueryStatus.Status.SUCCESSFUL,
+ QueryStatus.Status.CANCELED, QueryStatus.Status.CLOSED, QueryStatus.Status.FAILED);
+
+ /**
+ * Instantiates a new query ended.
+ *
+ * @param ctx
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ * @param user the user
+ * @param cause the cause
+ */
+ public QueryEnded(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
+ QueryHandle handle, String user, String cause) {
+ super(eventTime, prev, current, handle);
+ this.queryContext = ctx;
+ this.user = user;
+ this.cause = cause;
+ if (!END_STATES.contains(current)) {
+ throw new IllegalStateException("Not a valid end state: " + current + " query: " + handle);
+ }
+ }
+
+ public String toString() {
+ StringBuilder buf = new StringBuilder(super.toString());
+ if (StringUtils.isNotBlank(cause)) {
+ buf.append(" cause:").append(cause);
+ }
+ return buf.toString();
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEvent.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEvent.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEvent.java
new file mode 100644
index 0000000..f577b7d
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryEvent.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import java.util.UUID;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.server.api.events.LensEvent;
+
+import lombok.Getter;
+
+/**
+ * A generic event related to state change of a query Subclasses must declare the specific type of change they are
+ * interested in.
+ * <p></p>
+ * Every event will have an ID, which should be used by listeners to check if the event is already received.
+ *
+ * @param <T> Type of changed information about the query
+ */
+public abstract class QueryEvent<T> extends LensEvent {
+
+ /**
+ * The previous value.
+ */
+ @Getter
+ protected final T previousValue;
+
+ /**
+ * The current value.
+ */
+ @Getter
+ protected final T currentValue;
+
+ /**
+ * The query handle.
+ */
+ @Getter
+ protected final QueryHandle queryHandle;
+
+ /**
+ * The id.
+ */
+ protected final UUID id = UUID.randomUUID();
+
+ /**
+ * Instantiates a new query event.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QueryEvent(long eventTime, T prev, T current, QueryHandle handle) {
+ super(eventTime);
+ previousValue = prev;
+ currentValue = current;
+ this.queryHandle = handle;
+ }
+
+ @Override
+ public String getEventId() {
+ return id.toString();
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see java.lang.Object#toString()
+ */
+ @Override
+ public String toString() {
+ StringBuilder buf = new StringBuilder("QueryEvent: ").append(getClass().getSimpleName()).append(":{id: ")
+ .append(id).append(", query:").append(getQueryHandle()).append(", change:[").append(previousValue)
+ .append(" -> ").append(currentValue).append("]}");
+ return buf.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryExecuted.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryExecuted.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryExecuted.java
new file mode 100644
index 0000000..b4f2b37
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryExecuted.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * Event fired when query is successfully completed by the driver.
+ */
+public class QueryExecuted extends StatusChange {
+
+ /**
+ * Instantiates a new query executed.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QueryExecuted(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ checkCurrentState(QueryStatus.Status.EXECUTED);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryFailed.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryFailed.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryFailed.java
new file mode 100644
index 0000000..14a868a
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryFailed.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ * Event fired when a query fails to execute. Use getCause() to get the reason for failure.
+ */
+public class QueryFailed extends QueryEnded {
+
+ /**
+ * Instantiates a new query failed.
+ *
+ * @param ctx the query context
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ * @param user the user
+ * @param cause the cause
+ */
+ public QueryFailed(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
+ QueryHandle handle,
+ String user, String cause) {
+ super(ctx, eventTime, prev, current, handle, user, cause);
+ checkCurrentState(QueryStatus.Status.FAILED);
+ }
+
+ public QueryFailed(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState, String cause) {
+ this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle(), ctx.getSubmittedUser(), cause);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryLaunched.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryLaunched.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryLaunched.java
new file mode 100644
index 0000000..11f76da
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryLaunched.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * Event fired when query is LAUNCHED.
+ */
+public class QueryLaunched extends StatusChange {
+
+ /**
+ * Instantiates a new query launched.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QueryLaunched(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ checkCurrentState(QueryStatus.Status.LAUNCHED);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueued.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueued.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueued.java
new file mode 100644
index 0000000..57bd817
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueued.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * Event fired when a query is QUEUED.
+ */
+public class QueryQueued extends StatusChange {
+
+ /**
+ * The user.
+ */
+ private final String user;
+
+ /**
+ * Instantiates a new query queued.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ * @param user the user
+ */
+ public QueryQueued(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle,
+ String user) {
+ super(eventTime, prev, current, handle);
+ checkCurrentState(QueryStatus.Status.QUEUED);
+ this.user = user;
+ }
+
+ /**
+ * Get the submitting user
+ *
+ * @return user
+ */
+ public final String getUser() {
+ return user;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueuedForRetry.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueuedForRetry.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueuedForRetry.java
new file mode 100644
index 0000000..e90e743
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryQueuedForRetry.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * Event fired when a query is QUEUED.
+ */
+public class QueryQueuedForRetry extends QueryQueued {
+
+ /**
+ * Instantiates a new query queued.
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ * @param user the user
+ */
+ public QueryQueuedForRetry(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle,
+ String user) {
+ super(eventTime, prev, current, handle, user);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRejected.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRejected.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRejected.java
new file mode 100644
index 0000000..1a0b867
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRejected.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+
+/**
+ * The Class QueryRejected.
+ */
+public class QueryRejected extends QueryEvent<String> {
+
+ /**
+ * Instantiates a new query rejected.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QueryRejected(long eventTime, String prev, String current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRunning.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRunning.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRunning.java
new file mode 100644
index 0000000..c865412
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueryRunning.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+
+/**
+ * Event fired when query enters a RUNNING state.
+ */
+public class QueryRunning extends StatusChange {
+
+ /**
+ * Instantiates a new query running.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QueryRunning(long eventTime, QueryStatus.Status prev, QueryStatus.Status current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ checkCurrentState(QueryStatus.Status.RUNNING);
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QuerySuccess.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QuerySuccess.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QuerySuccess.java
new file mode 100644
index 0000000..9e08ba4
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QuerySuccess.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
+import org.apache.lens.server.api.query.QueryContext;
+
+/**
+ * Event fired when query is successfully completed.
+ */
+public class QuerySuccess extends QueryEnded {
+
+ /**
+ * Instantiates a new query success.
+ *
+ * @param ctx the query context
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QuerySuccess(QueryContext ctx, long eventTime, QueryStatus.Status prev, QueryStatus.Status current,
+ QueryHandle handle) {
+ super(ctx, eventTime, prev, current, handle, null, null);
+ checkCurrentState(QueryStatus.Status.SUCCESSFUL);
+ }
+
+ public QuerySuccess(QueryContext ctx, QueryStatus.Status prevState, QueryStatus.Status currState) {
+ this(ctx, ctx.getEndTime(), prevState, currState, ctx.getQueryHandle());
+ }
+}
http://git-wip-us.apache.org/repos/asf/lens/blob/38ab6c60/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueuePositionChange.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueuePositionChange.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueuePositionChange.java
new file mode 100644
index 0000000..bbc6f4f
--- /dev/null
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/events/QueuePositionChange.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.lens.server.api.query.events;
+
+import org.apache.lens.api.query.QueryHandle;
+
+/**
+ * Event fired when query moves up or down in the execution engine's queue.
+ */
+public class QueuePositionChange extends QueryEvent<Integer> {
+
+ /**
+ * Instantiates a new queue position change.
+ *
+ * @param eventTime the event time
+ * @param prev the prev
+ * @param current the current
+ * @param handle the handle
+ */
+ public QueuePositionChange(long eventTime, Integer prev, Integer current, QueryHandle handle) {
+ super(eventTime, prev, current, handle);
+ }
+}