You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2016/02/17 10:00:23 UTC
lens git commit: LENS-931 : Add unit test for fetch results being
called before query completion
Repository: lens
Updated Branches:
refs/heads/master c1e4434fd -> c4fa10723
LENS-931 : Add unit test for fetch results being called before query completion
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/c4fa1072
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/c4fa1072
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/c4fa1072
Branch: refs/heads/master
Commit: c4fa10723e094b07c6a0ba4dc3b2e490a0ed189c
Parents: c1e4434
Author: Puneet Gupta <pu...@gmail.com>
Authored: Wed Feb 17 14:30:04 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Wed Feb 17 14:30:04 2016 +0530
----------------------------------------------------------------------
.../lens/server/query/TestQueryService.java | 62 +++++++++++++++-----
1 file changed, 47 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/c4fa1072/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 737c99a..699fa68 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -46,14 +46,13 @@ import org.apache.lens.api.result.LensErrorTO;
import org.apache.lens.api.result.QueryCostTO;
import org.apache.lens.cube.error.LensCubeErrorCode;
import org.apache.lens.driver.hive.HiveDriver;
+import org.apache.lens.lib.query.FilePersistentFormatter;
import org.apache.lens.lib.query.FileSerdeFormatter;
import org.apache.lens.server.LensJerseyTest;
import org.apache.lens.server.LensServerTestUtil;
import org.apache.lens.server.LensServices;
import org.apache.lens.server.api.LensConfConstants;
-import org.apache.lens.server.api.driver.InMemoryResultSet;
-import org.apache.lens.server.api.driver.LensDriver;
-import org.apache.lens.server.api.driver.LensResultSetMetadata;
+import org.apache.lens.server.api.driver.*;
import org.apache.lens.server.api.error.LensDriverErrorCode;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.metrics.LensMetricsRegistry;
@@ -697,7 +696,8 @@ public class TestQueryService extends LensJerseyTest {
fail("unexpected cancel status: " + result.getStatus());
}
- // Test http download end point
+ // 1. Test http download end point and result path should be correct (when both driver and server persist)
+ // 2. Test Fetch result should fail before query is marked successful
log.info("Starting httpendpoint test");
final FormDataMultiPart mp3 = new FormDataMultiPart();
mp3.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), lensSessionId,
@@ -707,14 +707,31 @@ public class TestQueryService extends LensJerseyTest {
mp3.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("operation").build(), "execute"));
LensConf conf = new LensConf();
conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_SET, "true");
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true");
+ conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, DeferredPersistentResultFormatter.class.getName());
+ conf.addProperty("deferPersistenceByMillis", 5000); // defer persistence for 5 secs
mp3.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
mt));
final QueryHandle handle3 = target.request(mt).post(Entity.entity(mp3, MediaType.MULTIPART_FORM_DATA_TYPE),
new GenericType<LensAPIResult<QueryHandle>>() {}).getData();
- // Get query
+ QueryContext ctx3 = queryService.getQueryContext(handle3);
+ assertFalse(ctx3.finished()); //Formatting is deferred so query will take time to finish
+ try {
+ queryService.fetchResultSet(lensSessionId, handle3, 0, 100);
+ fail("client should not be allowed to fetch result before query finishes successfully");
+ } catch (NotFoundException e) {
+ // Expected. Ignore
+ }
waitForQueryToFinish(target(), lensSessionId, handle3, Status.SUCCESSFUL, mt);
+ LensResultSet rs = queryService.getResultset(handle3);
+ //check persisted result path
+ String expectedPath =
+ ctx3.getConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + handle3.getHandleIdString()
+ + ctx3.getConf().get(LensConfConstants.QUERY_OUTPUT_FILE_EXTN);
+ assertTrue(((PersistentResultSet) rs).getOutputPath().endsWith(expectedPath));
+
validateHttpEndPoint(target(), null, handle3, null);
}
@@ -1292,7 +1309,7 @@ public class TestQueryService extends LensJerseyTest {
conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "false");
conf.addProperty(LensConfConstants.PREFETCH_INMEMORY_RESULTSET, "true");
conf.addProperty(LensConfConstants.PREFETCH_INMEMORY_RESULTSET_ROWS, preFetchRows);
- conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, DeferredFileSerdeFormatter.class.getName());
+ conf.addProperty(LensConfConstants.QUERY_OUTPUT_FORMATTER, DeferredInMemoryResultFormatter.class.getName());
conf.addProperty("deferPersistenceByMillis", deferPersistenceByMillis); // property used for test only
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("conf").fileName("conf").build(), conf,
MediaType.APPLICATION_XML_TYPE));
@@ -1324,21 +1341,36 @@ public class TestQueryService extends LensJerseyTest {
false, true, MediaType.APPLICATION_XML_TYPE);
}
- private static class DeferredFileSerdeFormatter extends FileSerdeFormatter {
+ private static class DeferredInMemoryResultFormatter extends FileSerdeFormatter {
/**
* Defer init so that this output formatter takes significant time.
*/
@Override
public void init(QueryContext ctx, LensResultSetMetadata metadata) throws IOException {
super.init(ctx, metadata);
- long deferPersistenceByMillis = ctx.getConf().getLong("deferPersistenceByMillis", 5000);
- if (deferPersistenceByMillis > 0) {
- try {
- log.info("Deferring result formatting by {} millis", deferPersistenceByMillis);
- Thread.sleep(deferPersistenceByMillis);
- } catch (InterruptedException e) {
- // Ignore
- }
+ deferFormattingIfApplicable(ctx);
+ }
+ }
+
+ private static class DeferredPersistentResultFormatter extends FilePersistentFormatter {
+ /**
+ * Defer init so that this output formatter takes significant time.
+ */
+ @Override
+ public void init(QueryContext ctx, LensResultSetMetadata metadata) throws IOException {
+ super.init(ctx, metadata);
+ deferFormattingIfApplicable(ctx);
+ }
+ }
+
+ private static void deferFormattingIfApplicable(QueryContext ctx) {
+ long deferPersistenceByMillis = ctx.getConf().getLong("deferPersistenceByMillis", 0);
+ if (deferPersistenceByMillis > 0) {
+ try {
+ log.info("Deferring result formatting by {} millis", deferPersistenceByMillis);
+ Thread.sleep(deferPersistenceByMillis);
+ } catch (InterruptedException e) {
+ // Ignore
}
}
}