You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/08/11 14:20:55 UTC
[32/50] [abbrv] incubator-lens git commit: LENS-700: Execute Async
shall do semantic validation, rewrite,
select driver and estimate cost before accepting a query for asynchronous
execution
LENS-700: Execute Async shall do semantic validation, rewrite, select driver and estimate cost before accepting a query for asynchronous execution
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/06e5230c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/06e5230c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/06e5230c
Branch: refs/heads/current-release-line
Commit: 06e5230c6047019240fe8ae9f068e5702faa146d
Parents: eafc892
Author: Himanshu Gahlaut <ga...@gmail.com>
Authored: Wed Jul 15 15:10:41 2015 +0530
Committer: Himanshu Gahlaut <ga...@gmail.com>
Committed: Wed Aug 5 11:09:41 2015 +0530
----------------------------------------------------------------------
.../lens/server/api/driver/MockDriver.java | 1 +
.../server/query/QueryExecutionServiceImpl.java | 20 ++--
.../org/apache/lens/server/TestServerMode.java | 4 +-
.../apache/lens/server/TestServerRestart.java | 85 +++++++++-------
.../lens/server/common/FailingQueryDriver.java | 45 ++++++++
.../lens/server/common/RestAPITestUtil.java | 46 +++++++--
.../server/query/QueryAPIErrorResponseTest.java | 6 +-
.../server/query/TestQueryEndEmailNotifier.java | 71 +++++++------
.../lens/server/query/TestQueryService.java | 102 ++++++++-----------
.../resources/failing-query-driver-site.xml | 27 +++++
lens-server/src/test/resources/lens-site.xml | 3 +-
11 files changed, 259 insertions(+), 151 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index ea24336..17dc99b 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -94,6 +94,7 @@ public class MockDriver implements LensDriver {
public void configure(Configuration conf) throws LensException {
this.conf = conf;
ioTestVal = conf.getInt("mock.driver.test.val", -1);
+ this.conf.addResource("failing-query-driver-site.xml");
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index ae6d682..a7c0669 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -555,12 +555,6 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
try {
// acquire session before any query operation.
acquire(query.getLensSessionIdentifier());
- // the check to see if the query was already rewritten and selected driver's rewritten query is set
- if (!query.isDriverQueryExplicitlySet()) {
- rewriteAndSelect(query);
- } else {
- log.debug("Query is already rewritten");
- }
/* Check javadoc of QueryExecutionServiceImpl#removalFromLaunchedQueriesLock for reason for existence
of this lock. */
@@ -1595,7 +1589,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
} else {
ctx.setQueryName(pctx.getQueryName());
}
- return executeAsyncInternal(sessionHandle, ctx);
+ ctx.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
+ return submitQuery(ctx);
} finally {
release(sessionHandle);
}
@@ -1692,7 +1687,14 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
* @throws LensException the lens exception
*/
private QueryHandle executeAsyncInternal(LensSessionHandle sessionHandle, QueryContext ctx) throws LensException {
+
ctx.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
+ rewriteAndSelect(ctx);
+ return submitQuery(ctx);
+ }
+
+ private QueryHandle submitQuery(final QueryContext ctx) throws LensException {
+
QueryStatus before = ctx.getStatus();
ctx.setStatus(new QueryStatus(0.0, QUEUED, "Query is queued", false, null, null, null));
queuedQueries.add(ctx);
@@ -1848,6 +1850,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
accept(query, qconf, SubmitOp.EXECUTE);
QueryContext ctx = createContext(query, getSession(sessionHandle).getLoggedInUser(), conf, qconf);
ctx.setQueryName(queryName);
+ ctx.setLensSessionIdentifier(sessionHandle.getPublicId().toString());
+ rewriteAndSelect(ctx);
return executeTimeoutInternal(sessionHandle, ctx, timeoutMillis, qconf);
} finally {
release(sessionHandle);
@@ -1866,7 +1870,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
*/
private QueryHandleWithResultSet executeTimeoutInternal(LensSessionHandle sessionHandle, QueryContext ctx,
long timeoutMillis, Configuration conf) throws LensException {
- QueryHandle handle = executeAsyncInternal(sessionHandle, ctx);
+ QueryHandle handle = submitQuery(ctx);
QueryHandleWithResultSet result = new QueryHandleWithResultSet(handle);
// getQueryContext calls updateStatus, which fires query events if there's a change in status
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
index 28ff920..def5683 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerMode.java
@@ -37,6 +37,7 @@ import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.api.result.LensAPIResult;
import org.apache.lens.server.LensServices.SERVICE_MODE;
+import org.apache.lens.server.common.RestAPITestUtil;
import org.glassfish.jersey.media.multipart.FormDataBodyPart;
import org.glassfish.jersey.media.multipart.FormDataContentDisposition;
@@ -60,6 +61,7 @@ public class TestServerMode extends LensAllApplicationJerseyTest {
@BeforeTest
public void setUp() throws Exception {
super.setUp();
+ LensTestUtil.createTable("test_table", target(), RestAPITestUtil.openFooBarSession(target()));
}
/*
@@ -173,7 +175,7 @@ public class TestServerMode extends LensAllApplicationJerseyTest {
query.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
MediaType.APPLICATION_XML_TYPE));
- query.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select name from table"));
+ query.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select id from test_table"));
query.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
query.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
new LensConf(), MediaType.APPLICATION_XML_TYPE));
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
index b5235ad..4a88055 100644
--- a/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
+++ b/lens-server/src/test/java/org/apache/lens/server/TestServerRestart.java
@@ -18,6 +18,10 @@
*/
package org.apache.lens.server;
+import static org.apache.lens.server.common.RestAPITestUtil.execute;
+
+import static org.testng.Assert.assertEquals;
+
import java.io.*;
import java.util.*;
@@ -25,6 +29,7 @@ import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import org.apache.lens.api.*;
import org.apache.lens.api.APIResult.Status;
@@ -49,6 +54,7 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import com.google.common.base.Optional;
import lombok.extern.slf4j.Slf4j;
/**
@@ -191,13 +197,12 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
stat = ctx.getStatus();
Thread.sleep(1000);
}
- Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, "Expected to be successful "
- + handle);
+ assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.SUCCESSFUL, "Expected to be successful " + handle);
PersistentQueryResult resultset = target.path(handle.toString()).path("resultset")
.queryParam("sessionid", lensSessionId).request().get(PersistentQueryResult.class);
List<String> rows = TestQueryService.readResultSet(resultset, handle, true);
- Assert.assertEquals(rows.size(), 1);
- Assert.assertEquals(rows.get(0), "" + NROWS);
+ assertEquals(rows.size(), 1);
+ assertEquals(rows.get(0), "" + NROWS);
log.info("Completed {}", handle);
} catch (Exception exc) {
log.error("Failed query {}", handle, exc);
@@ -260,6 +265,16 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
Assert.assertNotNull(handle);
+ // wait for query to move out of QUEUED state
+ LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
+ .get(LensQuery.class);
+ QueryStatus stat = ctx.getStatus();
+ while (stat.queued()) {
+ ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
+ stat = ctx.getStatus();
+ Thread.sleep(1000);
+ }
+
List<LensSessionImpl.ResourceEntry> sessionResources = queryService.getSession(lensSessionId)
.getLensSessionPersistInfo().getResources();
int[] restoreCounts = new int[sessionResources.size()];
@@ -288,11 +303,10 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
// Check params to be set
verifyParamOnRestart(lensSessionId);
-
// Poll for first query, we should not get any exception
- LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
+ ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
- QueryStatus stat = ctx.getStatus();
+ stat = ctx.getStatus();
while (!stat.finished()) {
log.info("Polling query {} Status:{}", handle, stat);
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
@@ -303,37 +317,36 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
Assert.assertTrue(stat.finished());
log.info("Previous query status: {}", stat.getStatusMessage());
- for (int i = 0; i < 5; i++) {
- // Submit another query, again no exception expected
- mp = new FormDataMultiPart();
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
- MediaType.APPLICATION_XML_TYPE));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(),
- "select COUNT(ID) from test_hive_server_restart"));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(),
- new LensConf(), MediaType.APPLICATION_XML_TYPE));
- handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- Assert.assertNotNull(handle);
+ // After hive server restart, first few queries fail with Invalid Operation Handle followed by
+ // Invalid Session Handle. Idle behaviour is to fail with Invalid Session Handle immediately.
+ // Jira Ticket raised for debugging: https://issues.apache.org/jira/browse/LENS-707
- // Poll for second query, this should finish successfully
+ final String query = "select COUNT(ID) from test_hive_server_restart";
+ Response response = null;
+ while (response == null || response.getStatus() == Response.Status.INTERNAL_SERVER_ERROR.getStatusCode()) {
+ response = execute(target(), Optional.of(lensSessionId), Optional.of(query));
+ Thread.sleep(1000);
+ }
+
+ handle = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
+
+ // Poll for second query, this should finish successfully
+ ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
+ stat = ctx.getStatus();
+ while (!stat.finished()) {
+ log.info("Post restart polling query {} Status:{}", handle, stat);
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
stat = ctx.getStatus();
- while (!stat.finished()) {
- log.info("Post restart polling query {} Status:{}", handle, stat);
- ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
- stat = ctx.getStatus();
- Thread.sleep(1000);
- }
- log.info("@@ {} Final status for {}: {}", i, handle, stat.getStatus());
+ Thread.sleep(1000);
}
+ log.info("Final status for {}: {}", handle, stat.getStatus());
// Now we can expect that session resources have been added back exactly once
for (int i = 0; i < sessionResources.size(); i++) {
LensSessionImpl.ResourceEntry resourceEntry = sessionResources.get(i);
- Assert.assertEquals(resourceEntry.getRestoreCount(), 1 + restoreCounts[i], "Restore test failed for "
- + resourceEntry + " pre count=" + restoreCounts[i] + " post count=" + resourceEntry.getRestoreCount());
+ assertEquals(resourceEntry.getRestoreCount(), 1 + restoreCounts[i],
+ "Restore test failed for " + resourceEntry + " pre count=" + restoreCounts[i] + " post count=" + resourceEntry
+ .getRestoreCount());
log.info("@@ Latest count {}={}", resourceEntry, resourceEntry.getRestoreCount());
}
// Assert.assertEquals(stat.getStatus(), QueryStatus.Status.SUCCESSFUL,
@@ -378,7 +391,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
"target/test-classes/lens-site.xml"));
APIResult result = resourcetarget.path("add").request()
.put(Entity.entity(mp1, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
- Assert.assertEquals(result.getStatus(), Status.SUCCEEDED);
+ assertEquals(result.getStatus(), Status.SUCCEEDED);
// restart server
restartLensServer();
@@ -390,14 +403,14 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
Assert.assertTrue(sessionService.getHealthStatus().isHealthy());
LensSessionImpl session = sessionService.getSession(restartTestSession);
- Assert.assertEquals(session.getLensSessionPersistInfo().getResources().size(), 1);
+ assertEquals(session.getLensSessionPersistInfo().getResources().size(), 1);
LensSessionImpl.ResourceEntry resourceEntry = session.getLensSessionPersistInfo().getResources().get(0);
- Assert.assertEquals(resourceEntry.getType(), "file");
+ assertEquals(resourceEntry.getType(), "file");
Assert.assertTrue(resourceEntry.getLocation().contains("target/test-classes/lens-site.xml"));
// close session
result = sessionTarget.queryParam("sessionid", restartTestSession).request().delete(APIResult.class);
- Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
}
private void setParams(LensSessionHandle lensSessionHandle) {
@@ -409,7 +422,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
setpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("value").build(), "myvalue"));
APIResult result = target().path("session").path("params").request()
.put(Entity.entity(setpart, MediaType.MULTIPART_FORM_DATA_TYPE), APIResult.class);
- Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
}
private void verifyParamOnRestart(LensSessionHandle lensSessionHandle) {
@@ -417,7 +430,7 @@ public class TestServerRestart extends LensAllApplicationJerseyTest {
StringList sessionParams = target().path("session").path("params").queryParam("sessionid", lensSessionHandle)
.queryParam("verbose", true).queryParam("key", "lens.session.testRestartKey").request().get(StringList.class);
System.out.println("Session params:" + sessionParams.getElements());
- Assert.assertEquals(sessionParams.getElements().size(), 1);
+ assertEquals(sessionParams.getElements().size(), 1);
Assert.assertTrue(sessionParams.getElements().contains("lens.session.testRestartKey=myvalue"));
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java b/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java
new file mode 100644
index 0000000..64a75ce
--- /dev/null
+++ b/lens-server/src/test/java/org/apache/lens/server/common/FailingQueryDriver.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.lens.server.common;
+
+import org.apache.lens.server.api.driver.MockDriver;
+import org.apache.lens.server.api.error.LensException;
+import org.apache.lens.server.api.query.AbstractQueryContext;
+import org.apache.lens.server.api.query.QueryContext;
+import org.apache.lens.server.api.query.cost.FactPartitionBasedQueryCost;
+import org.apache.lens.server.api.query.cost.QueryCost;
+
+public class FailingQueryDriver extends MockDriver {
+
+ @Override
+ public QueryCost estimate(final AbstractQueryContext ctx) throws LensException {
+
+ if (ctx.getUserQuery().contains("fail")) {
+ return new FactPartitionBasedQueryCost(0.0);
+ } else {
+ throw new LensException();
+ }
+ }
+
+ @Override
+ public void executeAsync(final QueryContext context) throws LensException {
+ throw new LensException("Simulated Launch Failure");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
index 80e29ba..0945598 100644
--- a/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
+++ b/lens-server/src/test/java/org/apache/lens/server/common/RestAPITestUtil.java
@@ -34,6 +34,9 @@ import org.apache.lens.api.LensSessionHandle;
import org.apache.lens.api.metastore.ObjectFactory;
import org.apache.lens.api.metastore.XCube;
import org.apache.lens.api.metastore.XFactTable;
+import org.apache.lens.api.query.LensQuery;
+import org.apache.lens.api.query.QueryHandle;
+import org.apache.lens.api.query.QueryStatus;
import org.glassfish.jersey.media.multipart.FormDataMultiPart;
@@ -47,6 +50,9 @@ public class RestAPITestUtil {
throw new UnsupportedOperationException();
}
+ public static LensSessionHandle openFooBarSession(final WebTarget target) {
+ return openSession(target, "foo", "bar");
+ }
public static LensSessionHandle openSession(final WebTarget target, final String userName, final String passwd) {
return openSession(target, userName, passwd, new LensConf());
}
@@ -58,33 +64,34 @@ public class RestAPITestUtil {
Optional.of(userName), Optional.of(passwd), Optional.of(conf));
return target.path("session").request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- LensSessionHandle.class);
+ LensSessionHandle.class);
}
public static Response estimate(final WebTarget target, final Optional<LensSessionHandle> sessionId,
final Optional<String> query) {
- return runQuery(target, sessionId, query, Optional.of("estimate"));
+ return postQuery(target, sessionId, query, Optional.of("estimate"));
}
- public static Response explain(final WebTarget target, final Optional<LensSessionHandle> sessionId,
+
+ public static Response execute(final WebTarget target, final Optional<LensSessionHandle> sessionId,
final Optional<String> query) {
- return runQuery(target, sessionId, query, Optional.of("explain"));
+ return postQuery(target, sessionId, query, Optional.of("execute"));
}
- public static Response runQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId,
+ public static Response postQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId,
final Optional<String> query, final Optional<String> operation) {
- return runQuery(target, sessionId, query, operation, new LensConf());
+ return postQuery(target, sessionId, query, operation, new LensConf());
}
- public static Response runQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId,
+ public static Response postQuery(final WebTarget target, final Optional<LensSessionHandle> sessionId,
final Optional<String> query, final Optional<String> operation, final LensConf conf) {
FormDataMultiPart mp = FormDataMultiPartFactory
.createFormDataMultiPartForQuery(sessionId, query, operation, conf);
- return target.path("queryapi/queries").request(MediaType.APPLICATION_XML).post(Entity.entity(mp,
- MediaType.MULTIPART_FORM_DATA_TYPE));
+ return target.path("queryapi/queries").request(MediaType.APPLICATION_XML).post(
+ Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE));
}
public static void closeSessionFailFast(final WebTarget target, final LensSessionHandle sessionId) {
@@ -181,4 +188,25 @@ public class RestAPITestUtil {
throw new RuntimeException("Setup failed");
}
}
+
+ public static LensQuery waitForQueryToFinish(final WebTarget target, final LensSessionHandle lensSessionHandle,
+ final QueryHandle handle) throws InterruptedException {
+
+ LensQuery lensQuery = getLensQuery(target, lensSessionHandle, handle);
+
+ QueryStatus stat = lensQuery.getStatus();
+ while (!stat.finished()) {
+ lensQuery = getLensQuery(target, lensSessionHandle, handle);
+ stat = lensQuery.getStatus();
+ Thread.sleep(1000);
+ }
+ return lensQuery;
+ }
+
+ public static LensQuery getLensQuery(final WebTarget target, final LensSessionHandle lensSessionHandle,
+ final QueryHandle handle) {
+
+ return target.path("queryapi/queries").path(handle.toString()).queryParam("sessionid", lensSessionHandle).request()
+ .get(LensQuery.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
index 91e9081..9a45512 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/QueryAPIErrorResponseTest.java
@@ -139,7 +139,7 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
LensSessionHandle sessionId = openSession(target(), "foo", "bar", new LensConf());
- Response response = runQuery(target(), Optional.of(sessionId), Optional.of(MOCK_QUERY),
+ Response response = postQuery(target(), Optional.of(sessionId), Optional.of(MOCK_QUERY),
Optional.of(INVALID_OPERATION));
final String expectedErrMsg = "Provided Operation is not supported. Supported Operations are: "
@@ -166,9 +166,11 @@ public class QueryAPIErrorResponseTest extends LensJerseyTest {
expectedErrMsg, MOCK_STACK_TRACE);
LensErrorTO childError2 = LensErrorTO.composedOf(INTERNAL_SERVER_ERROR.getValue(),
expectedErrMsg, MOCK_STACK_TRACE);
+ LensErrorTO childError3 = LensErrorTO.composedOf(INTERNAL_SERVER_ERROR.getValue(),
+ expectedErrMsg, MOCK_STACK_TRACE);
LensErrorTO expectedLensErrorTO = LensErrorTO.composedOf(INTERNAL_SERVER_ERROR.getValue(),
- expectedErrMsg, MOCK_STACK_TRACE, Arrays.asList(childError1, childError2));
+ expectedErrMsg, MOCK_STACK_TRACE, Arrays.asList(childError1, childError2, childError3));
ErrorResponseExpectedData expectedData = new ErrorResponseExpectedData(Status.INTERNAL_SERVER_ERROR,
expectedLensErrorTO);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
index f1461bc..deb7642 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryEndEmailNotifier.java
@@ -18,6 +18,11 @@
*/
package org.apache.lens.server.query;
+import static org.apache.lens.server.common.RestAPITestUtil.execute;
+import static org.apache.lens.server.common.RestAPITestUtil.waitForQueryToFinish;
+
+import static org.testng.Assert.assertEquals;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -28,6 +33,7 @@ import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.Application;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import org.apache.lens.api.LensConf;
import org.apache.lens.api.LensSessionHandle;
@@ -56,6 +62,8 @@ import org.testng.annotations.AfterTest;
import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import com.google.common.base.Optional;
+
/**
* The query completion email notifier
*/
@@ -194,16 +202,22 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest {
@Test
public void testEmailNotification() throws InterruptedException {
wiser.start();
+
LensConf conf = new LensConf();
+
// launch failure
- QueryHandle handle = launchAndWaitForQuery(conf, "select ID from non_exist_table", QueryStatus.Status.FAILED);
+ final Response response = execute(target(), Optional.of(lensSessionId), Optional.of("select fail from non_exist"));
+ assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
+ QueryHandle handle = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
+ LensQuery lensQuery = waitForQueryToFinish(target(), lensSessionId, handle);
+
List<WiserMessage> messages = new ArrayList<WiserMessage>();
for (int i = 0; i < NUM_ITERS; i++) {
messages = wiser.getMessages();
if (messages.size() >= 4) {
break;
}
- Thread.sleep(10000);
+ Thread.sleep(2000);
}
Assert.assertEquals(messages.size(), 4);
@@ -211,22 +225,6 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest {
Assert.assertTrue(messages.get(0).toString().contains("Launching query failed"));
Assert.assertTrue(messages.get(0).toString().contains("Reason"));
- // rewriter failure
- handle = launchAndWaitForQuery(conf, "cube select ID from nonexist", QueryStatus.Status.FAILED);
- messages = new ArrayList<WiserMessage>();
- for (int i = 0; i < NUM_ITERS; i++) {
- messages = wiser.getMessages();
- if (messages.size() >= 8) {
- break;
- }
- Thread.sleep(10000);
- }
-
- Assert.assertEquals(messages.size(), 8);
- Assert.assertTrue(messages.get(4).toString().contains(handle.toString()));
- Assert.assertTrue(messages.get(4).toString().contains("Launching query failed"));
- Assert.assertTrue(messages.get(4).toString().contains("Reason"));
-
// formatting failure
conf = new LensConf();
conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true");
@@ -234,19 +232,20 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest {
conf.addProperty(LensConfConstants.QUERY_OUTPUT_SERDE, "NonexistentSerde.class");
handle = launchAndWaitForQuery(conf, "select ID, IDSTR from " + TEST_TABLE,
QueryStatus.Status.FAILED);
+
messages = new ArrayList<WiserMessage>();
for (int i = 0; i < NUM_ITERS; i++) {
messages = wiser.getMessages();
- if (messages.size() >= 12) {
+ if (messages.size() >= 8) {
break;
}
- Thread.sleep(10000);
+ Thread.sleep(2000);
}
- Assert.assertEquals(messages.size(), 12);
- Assert.assertTrue(messages.get(8).toString().contains(handle.toString()));
- Assert.assertTrue(messages.get(8).toString().contains("Result formatting failed!"));
- Assert.assertTrue(messages.get(8).toString().contains("Reason"));
+ Assert.assertEquals(messages.size(), 8);
+ Assert.assertTrue(messages.get(4).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(4).toString().contains("Result formatting failed!"));
+ Assert.assertTrue(messages.get(4).toString().contains("Reason"));
// execution failure
conf = new LensConf();
@@ -258,16 +257,16 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest {
messages = new ArrayList<WiserMessage>();
for (int i = 0; i < NUM_ITERS; i++) {
messages = wiser.getMessages();
- if (messages.size() >= 16) {
+ if (messages.size() >= 8) {
break;
}
- Thread.sleep(10000);
+ Thread.sleep(2000);
}
- Assert.assertEquals(messages.size(), 16);
- Assert.assertTrue(messages.get(12).toString().contains(handle.toString()));
- Assert.assertTrue(messages.get(12).toString().contains("Query execution failed!"));
- Assert.assertTrue(messages.get(12).toString().contains("Reason"));
+ Assert.assertEquals(messages.size(), 12);
+ Assert.assertTrue(messages.get(8).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(8).toString().contains("Query execution failed!"));
+ Assert.assertTrue(messages.get(8).toString().contains("Reason"));
// successful query
conf = new LensConf();
@@ -277,15 +276,15 @@ public class TestQueryEndEmailNotifier extends LensJerseyTest {
messages = new ArrayList<WiserMessage>();
for (int i = 0; i < NUM_ITERS; i++) {
messages = wiser.getMessages();
- if (messages.size() >= 20) {
+ if (messages.size() >= 16) {
break;
}
- Thread.sleep(10000);
+ Thread.sleep(2000);
}
- Assert.assertEquals(messages.size(), 20);
- Assert.assertTrue(messages.get(16).toString().contains(handle.toString()));
- Assert.assertTrue(messages.get(16).toString().contains("Query SUCCESSFUL"));
- Assert.assertTrue(messages.get(16).toString().contains("Result available at"));
+ Assert.assertEquals(messages.size(), 16);
+ Assert.assertTrue(messages.get(12).toString().contains(handle.toString()));
+ Assert.assertTrue(messages.get(12).toString().contains("Query SUCCESSFUL"));
+ Assert.assertTrue(messages.get(12).toString().contains("Result available at"));
wiser.stop();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 4b9962a..84b315c 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -20,6 +20,9 @@ package org.apache.lens.server.query;
import static javax.ws.rs.core.Response.Status.INTERNAL_SERVER_ERROR;
+import static org.apache.lens.server.common.RestAPITestUtil.execute;
+import static org.apache.lens.server.common.RestAPITestUtil.waitForQueryToFinish;
+
import static org.testng.Assert.*;
import java.io.*;
@@ -41,7 +44,6 @@ import org.apache.lens.api.query.*;
import org.apache.lens.api.query.QueryStatus.Status;
import org.apache.lens.api.result.*;
import org.apache.lens.driver.hive.HiveDriver;
-import org.apache.lens.driver.hive.TestHiveDriver;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.LensTestUtil;
@@ -79,6 +81,7 @@ import org.testng.annotations.Test;
import com.codahale.metrics.MetricRegistry;
+import com.google.common.base.Optional;
import lombok.extern.slf4j.Slf4j;
/**
@@ -213,48 +216,46 @@ public class TestQueryService extends LensJerseyTest {
}
/**
- * Test launch fail.
+ * Test rewrite failure in execute operation.
*
* @throws InterruptedException the interrupted exception
*/
@Test
- public void testLaunchFail() throws InterruptedException {
+ public void testRewriteFailureInExecute() throws InterruptedException {
final WebTarget target = target().path("queryapi/queries");
- long failedQueries = metricsSvc.getTotalFailedQueries();
- System.out.println("%% " + failedQueries);
LensConf conf = new LensConf();
final FormDataMultiPart mp = new FormDataMultiPart();
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
- MediaType.APPLICATION_XML_TYPE));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(),
- "select ID from non_exist_table"));
+ MediaType.APPLICATION_XML_TYPE));
+ mp.bodyPart(
+ new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID from non_exist_table"));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
- MediaType.APPLICATION_XML_TYPE));
- final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryHandle>>() {
- }).getData();
+ MediaType.APPLICATION_XML_TYPE));
+ final Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE));
+ assertEquals(response.getStatus(), Response.Status.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
- assertNotNull(handle);
+ /**
+ * Test launch failure in execute operation.
+ *
+ * @throws InterruptedException the interrupted exception
+ */
+ @Test
+ public void testLaunchFail() throws InterruptedException {
- LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
- .get(LensQuery.class);
- QueryStatus stat = ctx.getStatus();
- while (!stat.finished()) {
- ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
- stat = ctx.getStatus();
- System.out.println("%% query " + ctx.getQueryHandle() + " status:" + stat);
- Thread.sleep(1000);
- }
+ final Response response = execute(target(), Optional.of(lensSessionId), Optional.of("select fail from non_exist"));
+ assertEquals(response.getStatus(), Response.Status.OK.getStatusCode());
- assertTrue(ctx.getSubmissionTime() > 0);
- assertEquals(ctx.getLaunchTime(), 0);
- assertEquals(ctx.getDriverStartTime(), 0);
- assertEquals(ctx.getDriverFinishTime(), 0);
- assertTrue(ctx.getFinishTime() > 0);
- assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
- /* Commented and jira ticket raised for correction: https://issues.apache.org/jira/browse/LENS-685
- assertTrue(metricsSvc.getTotalFailedQueries() >= failedQueries + 1);*/
+ QueryHandle handle = response.readEntity(new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
+ LensQuery lensQuery = waitForQueryToFinish(target(), lensSessionId, handle);
+
+ assertTrue(lensQuery.getSubmissionTime() > 0);
+ assertEquals(lensQuery.getLaunchTime(), 0);
+ assertEquals(lensQuery.getDriverStartTime(), 0);
+ assertEquals(lensQuery.getDriverFinishTime(), 0);
+ assertTrue(lensQuery.getFinishTime() > 0);
+ assertEquals(lensQuery.getStatus().getStatus(), QueryStatus.Status.FAILED);
}
// test with execute async post, get all queries, get query context,
@@ -269,14 +270,12 @@ public class TestQueryService extends LensJerseyTest {
public void testQueriesAPI() throws InterruptedException {
// test post execute op
final WebTarget target = target().path("queryapi/queries");
- LensConf conf = new LensConf();
- conf.addProperty("hive.exec.driver.run.hooks", TestHiveDriver.FailHook.class.getCanonicalName());
final FormDataMultiPart mp = new FormDataMultiPart();
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
MediaType.APPLICATION_XML_TYPE));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("query").build(), "select ID from " + TEST_TABLE));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
- mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
+ mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(),
MediaType.APPLICATION_XML_TYPE));
long queuedQueries = metricsSvc.getQueuedQueries();
@@ -286,7 +285,6 @@ public class TestQueryService extends LensJerseyTest {
final QueryHandle handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {
}).getData();
-
assertNotNull(handle);
// Get all queries
@@ -295,38 +293,29 @@ public class TestQueryService extends LensJerseyTest {
.get(new GenericType<List<QueryHandle>>() {});
assertTrue(allQueriesXML.size() >= 1);
- // JSON
- // List<QueryHandle> allQueriesJSON = target.request(
- // MediaType.APPLICATION_JSON).get(new GenericType<List<QueryHandle>>() {
- // });
- // Assert.assertEquals(allQueriesJSON.size(), 1);
- // JAXB
List<QueryHandle> allQueries = (List<QueryHandle>) target.queryParam("sessionid", lensSessionId).request()
- .get(new GenericType<List<QueryHandle>>() {
- });
+ .get(new GenericType<List<QueryHandle>>() {});
assertTrue(allQueries.size() >= 1);
assertTrue(allQueries.contains(handle));
- // Get query
- // Invocation.Builder builderjson = target.path(handle.toString()).request(MediaType.APPLICATION_JSON);
- // String responseJSON = builderjson.get(String.class);
- // System.out.println("query JSON:" + responseJSON);
String queryXML = target.path(handle.toString()).queryParam("sessionid", lensSessionId)
.request(MediaType.APPLICATION_XML).get(String.class);
- System.out.println("query XML:" + queryXML);
+ log.debug("query XML:{}", queryXML);
Response response = target.path(handle.toString() + "001").queryParam("sessionid", lensSessionId).request().get();
assertEquals(response.getStatus(), 404);
LensQuery ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request()
.get(LensQuery.class);
- // Assert.assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.QUEUED);
// wait till the query finishes
QueryStatus stat = ctx.getStatus();
while (!stat.finished()) {
+ Thread.sleep(1000);
ctx = target.path(handle.toString()).queryParam("sessionid", lensSessionId).request().get(LensQuery.class);
stat = ctx.getStatus();
+ /*
+ Commented due to same issue as: https://issues.apache.org/jira/browse/LENS-683
switch (stat.getStatus()) {
case RUNNING:
assertEquals(metricsSvc.getRunningQueries(), runningQueries + 1);
@@ -335,16 +324,16 @@ public class TestQueryService extends LensJerseyTest {
assertEquals(metricsSvc.getQueuedQueries(), queuedQueries + 1);
break;
default: // nothing
- }
- Thread.sleep(1000);
+ }*/
}
+
assertTrue(ctx.getSubmissionTime() > 0);
assertTrue(ctx.getFinishTime() > 0);
- assertEquals(ctx.getStatus().getStatus(), QueryStatus.Status.FAILED);
+ assertEquals(ctx.getStatus().getStatus(), Status.SUCCESSFUL);
// Update conf for query
final FormDataMultiPart confpart = new FormDataMultiPart();
- conf = new LensConf();
+ LensConf conf = new LensConf();
conf.addProperty("my.property", "myvalue");
confpart.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
MediaType.APPLICATION_XML_TYPE));
@@ -1229,13 +1218,10 @@ public class TestQueryService extends LensJerseyTest {
// set a timeout value enough for tests
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("timeoutmillis").build(), "300000"));
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), new LensConf(),
- MediaType.APPLICATION_XML_TYPE));
+ MediaType.APPLICATION_XML_TYPE));
- QueryHandleWithResultSet result = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensAPIResult<QueryHandleWithResultSet>>() {}).getData();
- assertNotNull(result.getQueryHandle());
- assertNull(result.getResult());
- assertEquals(result.getStatus().getStatus(), Status.FAILED);
+ Response response = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE));
+ assertEquals(response.getStatus(), Response.Status.INTERNAL_SERVER_ERROR.getStatusCode());
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/resources/failing-query-driver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/failing-query-driver-site.xml b/lens-server/src/test/resources/failing-query-driver-site.xml
new file mode 100644
index 0000000..fee022d
--- /dev/null
+++ b/lens-server/src/test/resources/failing-query-driver-site.xml
@@ -0,0 +1,27 @@
+<?xml version="1.0"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing,
+ ~ software distributed under the License is distributed on an
+ ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ~ KIND, either express or implied. See the License for the
+ ~ specific language governing permissions and limitations
+ ~ under the License.
+ -->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<configuration>
+ <property>
+ <name>lens.driver.test.key</name>
+ <value>set</value>
+ </property>
+</configuration>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/06e5230c/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index bd93828..6179c41 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -24,7 +24,8 @@
<configuration>
<property>
<name>lens.server.drivers</name>
- <value>org.apache.lens.driver.hive.HiveDriver,org.apache.lens.driver.jdbc.JDBCDriver</value>
+ <value>org.apache.lens.driver.hive.HiveDriver,org.apache.lens.driver.jdbc.JDBCDriver,
+ org.apache.lens.server.common.FailingQueryDriver</value>
</property>
<property>