You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2015/05/22 07:02:28 UTC
incubator-lens git commit: LENS-26 : Add cli option to download the
result into a file ( Rajat Khandelwal via amareshwari)
Repository: incubator-lens
Updated Branches:
refs/heads/master 063a3766f -> d47af610c
LENS-26 : Add cli option to download the result into a file ( Rajat Khandelwal via amareshwari)
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/d47af610
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/d47af610
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/d47af610
Branch: refs/heads/master
Commit: d47af610cf329815795dd7151319bebf4a5394d5
Parents: 063a376
Author: Rajat Khandelwal <pr...@apache.org>
Authored: Fri May 22 10:32:13 2015 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Fri May 22 10:32:13 2015 +0530
----------------------------------------------------------------------
checkstyle/src/main/resources/checkstyle.xml | 6 +-
.../lens/cli/commands/BaseLensCommand.java | 11 +-
.../lens/cli/commands/LensCRUDCommand.java | 4 +-
.../LensCRUDStoragePartitionCommand.java | 6 +-
.../lens/cli/commands/LensQueryCommands.java | 58 +++++++--
.../apache/lens/cli/TestLensQueryCommands.java | 120 +++++++++++--------
.../lens/cli/doc/TestGenerateCLIUserDoc.java | 7 +-
.../java/org/apache/lens/client/LensClient.java | 57 +++++----
.../org/apache/lens/client/LensStatement.java | 32 ++++-
.../org/apache/lens/driver/hive/HiveDriver.java | 10 +-
.../lens/driver/hive/HiveInMemoryResultSet.java | 19 ++-
.../apache/lens/driver/jdbc/JDBCResultSet.java | 23 +++-
.../server/api/driver/InMemoryResultSet.java | 1 +
.../lens/server/api/driver/MockDriver.java | 5 +
.../server/query/QueryExecutionServiceImpl.java | 10 +-
src/site/apt/user/cli.apt | 2 +-
16 files changed, 250 insertions(+), 121 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/checkstyle/src/main/resources/checkstyle.xml
----------------------------------------------------------------------
diff --git a/checkstyle/src/main/resources/checkstyle.xml b/checkstyle/src/main/resources/checkstyle.xml
index 236f977..d6ff839 100644
--- a/checkstyle/src/main/resources/checkstyle.xml
+++ b/checkstyle/src/main/resources/checkstyle.xml
@@ -240,9 +240,9 @@
<!-- allow warnings to be suppressed -->
<module name="SuppressionCommentFilter">
- <property name="offCommentFormat" value="SUSPEND CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck"/>
- <property name="onCommentFormat" value="RESUME CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck"/>
- <property name="checkFormat" value="ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck"/>
+ <property name="offCommentFormat" value="SUSPEND CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck|InnerAssignmentCheck"/>
+ <property name="onCommentFormat" value="RESUME CHECKSTYLE CHECK ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck|InnerAssignmentCheck"/>
+ <property name="checkFormat" value="ParameterNumberCheck|VisibilityModifierCheck|HiddenFieldCheck|MethodName|HideUtilityClassConstructorCheck|DoubleCheckedLockingCheck|InnerAssignmentCheck"/>
</module>
<!--module name="SuppressionFilter">
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
index 8707fec..d6055cd 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/BaseLensCommand.java
@@ -140,14 +140,21 @@ public class BaseLensCommand implements ExecutionProcessor {
.replaceAll("]", "\n").replaceAll(",", "").replaceAll("\"", "").replaceAll("\n\n", "\n");
}
- public String getValidPath(String path) {
+ public String getValidPath(String path, boolean shouldBeDirectory, boolean shouldExist) {
+ path = path.replaceAll("/$", "");
if (path.startsWith("~")) {
path = path.replaceFirst("~", System.getProperty("user.home"));
}
File f = new File(path);
- if (!f.exists()) {
+ if (shouldExist && !f.exists()) {
throw new RuntimeException("Path " + path + " doesn't exist.");
}
+ if (shouldBeDirectory && !f.isDirectory()) {
+ throw new RuntimeException("Path " + path + " is not a directory");
+ }
+ if (!shouldBeDirectory && f.isDirectory()) {
+ throw new RuntimeException("Path " + path + " is a directory");
+ }
return f.getAbsolutePath();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java
index 05922e3..9cb7d69 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDCommand.java
@@ -38,7 +38,7 @@ public abstract class LensCRUDCommand<T> extends BaseLensCommand {
}
public String create(String path, boolean ignoreIfExists) {
- return doCreate(getValidPath(path), ignoreIfExists).getStatus().toString().toLowerCase();
+ return doCreate(getValidPath(path, false, true), ignoreIfExists).getStatus().toString().toLowerCase();
}
public String describe(String name) {
@@ -50,7 +50,7 @@ public abstract class LensCRUDCommand<T> extends BaseLensCommand {
}
public String update(String entity, String path) {
- return doUpdate(entity, getValidPath(path)).getStatus().toString().toLowerCase();
+ return doUpdate(entity, getValidPath(path, false, true)).getStatus().toString().toLowerCase();
}
public String drop(String name, boolean cascade) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java
index 0d53e21..208081d 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensCRUDStoragePartitionCommand.java
@@ -53,7 +53,7 @@ public abstract class LensCRUDStoragePartitionCommand<T> extends LensCRUDCommand
}
public String addStorage(String tableName, String path) {
- return doAddStorage(tableName, getValidPath(path)).toString().toLowerCase();
+ return doAddStorage(tableName, getValidPath(path, false, true)).toString().toLowerCase();
}
public String getStorage(String tableName, String storage) {
@@ -81,11 +81,11 @@ public abstract class LensCRUDStoragePartitionCommand<T> extends LensCRUDCommand
}
public String addPartition(String tableName, String storageName, String path) {
- return doAddPartition(tableName, storageName, getValidPath(path)).toString().toLowerCase();
+ return doAddPartition(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
}
public String addPartitions(String tableName, String storageName, String path) {
- return doAddPartitions(tableName, storageName, getValidPath(path)).toString().toLowerCase();
+ return doAddPartitions(tableName, storageName, getValidPath(path, false, true)).toString().toLowerCase();
}
public String dropPartitions(String tableName, String storageName, String filter) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 64e9ca4..6c4a1b5 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -18,18 +18,18 @@
*/
package org.apache.lens.cli.commands;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.UnsupportedEncodingException;
+import java.io.*;
import java.nio.charset.Charset;
import java.util.List;
import java.util.UUID;
+import javax.ws.rs.core.Response;
+
import org.apache.lens.api.query.*;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
import org.apache.lens.client.LensClient;
+import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.shell.core.annotation.CliCommand;
@@ -40,6 +40,7 @@ import com.google.common.base.Joiner;
/**
* The Class LensQueryCommands.
+ * SUSPEND CHECKSTYLE CHECK InnerAssignmentCheck
*/
@Component
@UserDocumentation(title = "Query Management",
@@ -200,10 +201,10 @@ public class LensQueryCommands extends BaseLensCommand {
return "Explain FAILED:" + plan.getErrorMsg();
}
if (StringUtils.isNotBlank(location)) {
- String validPath = getValidPath(location);
- OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(validPath), Charset.defaultCharset());
- osw.write(plan.getPlanString());
- osw.close();
+ String validPath = getValidPath(location, false, false);
+ try (OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(validPath), Charset.defaultCharset())) {
+ osw.write(plan.getPlanString());
+ }
return "Saved to " + validPath;
}
return plan.getPlanString();
@@ -262,13 +263,44 @@ public class LensQueryCommands extends BaseLensCommand {
* @param qh the qh
* @return the query results
*/
- @CliCommand(value = "query results", help = "get results of async query with query handle <query_handle>")
+ @CliCommand(value = "query results",
+ help = "get results of async query with query handle <query_handle>. Can "
+ + "optionally save the results to a file by providing <save_location>")
public String getQueryResults(
- @CliOption(key = {"", "query_handle"}, mandatory = true, help = "<query_handle>") String qh) {
+ @CliOption(key = {"", "query_handle"}, mandatory = true, help = "<query_handle>") String qh,
+ @CliOption(key = {"save_location"}, mandatory = false, help = "<save_location>") String location) {
+ QueryHandle queryHandle = new QueryHandle(UUID.fromString(qh));
try {
- LensClient.LensClientResultSetWithStats result = getClient()
- .getAsyncResults(new QueryHandle(UUID.fromString(qh)));
- return formatResultSet(result);
+ String prefix = "";
+ if (StringUtils.isNotBlank(location)) {
+ location = getValidPath(location, true, true);
+ Response response = getClient().getHttpResults(queryHandle);
+ if (response.getStatus() == Response.Status.OK.getStatusCode()) {
+ String disposition = (String) response.getHeaders().get("content-disposition").get(0);
+ String fileName = disposition.split("=")[1].trim();
+ location = getValidPath(location + File.separator + fileName, false, false);
+ try (InputStream stream = response.readEntity(InputStream.class);
+ FileOutputStream outStream = new FileOutputStream(new File(location))) {
+ IOUtils.copy(stream, outStream);
+ }
+ return "Saved to " + location;
+ } else {
+ LensClient.LensClientResultSetWithStats results = getClient().getAsyncResults(queryHandle);
+ if (results.getResultSet().getResult() instanceof InMemoryQueryResult) {
+ location = getValidPath(location + File.separator + qh + ".csv", false, false);
+ try (OutputStreamWriter osw = new OutputStreamWriter(new FileOutputStream(location),
+ Charset.defaultCharset())) {
+ osw.write(formatResultSet(results));
+ }
+ return "Saved to " + location;
+ } else {
+ return "Can't download the result because it's available in driver's persistence.\n"
+ + formatResultSet(results);
+ }
+ }
+ } else {
+ return formatResultSet(getClient().getAsyncResults(queryHandle));
+ }
} catch (Throwable t) {
return t.getMessage();
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
index d7b6372..5724d24 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/TestLensQueryCommands.java
@@ -18,9 +18,12 @@
*/
package org.apache.lens.cli;
-import java.io.File;
+import static org.testng.Assert.*;
+
+import java.io.*;
import java.net.URL;
import java.util.GregorianCalendar;
+import java.util.Scanner;
import java.util.UUID;
import javax.ws.rs.BadRequestException;
@@ -42,7 +45,6 @@ import org.apache.hadoop.fs.Path;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.testng.Assert;
import org.testng.annotations.Test;
/**
@@ -65,6 +67,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
+ " test_dim\n" + " name\n" + " TOK_WHERE\n" + " =\n"
+ " .\n" + " TOK_TABLE_OR_COL\n" + " test_dim\n"
+ " dt\n" + " 'latest'";
+ private File resDir;
/**
* Test query commands.
@@ -78,6 +81,8 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
setup(client);
LensQueryCommands qCom = new LensQueryCommands();
qCom.setClient(client);
+ resDir = new File("target/results");
+ assertTrue(resDir.exists() || resDir.mkdirs());
testExecuteSyncQuery(qCom);
testExecuteAsyncQuery(qCom);
testExplainQuery(qCom);
@@ -92,7 +97,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
client.setConnectionParam("lens.query.enable.metrics.per.query", "true");
qCom.setClient(client);
String result = qCom.getAllPreparedQueries("all", "", -1, -1);
- Assert.assertEquals(result, "No prepared queries");
+ assertEquals(result, "No prepared queries");
testExecuteSyncQuery(qCom);
testExecuteAsyncQuery(qCom);
testExplainQuery(qCom);
@@ -114,19 +119,19 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String sql = "cube select id, name from test_dim";
String result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, Long.MAX_VALUE);
- Assert.assertEquals(result, "No prepared queries");
+ assertEquals(result, "No prepared queries");
final String qh = qCom.prepare(sql, "testPreparedName");
result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, System.currentTimeMillis());
- Assert.assertEquals(qh, result);
+ assertEquals(qh, result);
result = qCom.getPreparedStatus(qh);
- Assert.assertTrue(result.contains("User query:cube select id, name from test_dim"));
- Assert.assertTrue(result.contains(qh));
+ assertTrue(result.contains("User query:cube select id, name from test_dim"));
+ assertTrue(result.contains(qh));
result = qCom.executePreparedQuery(qh, false, "testPrepQuery1");
LOG.warn("XXXXXX Prepared query sync result is " + result);
- Assert.assertTrue(result.contains("1\tfirst"));
+ assertTrue(result.contains("1\tfirst"));
String handle = qCom.executePreparedQuery(qh, true, "testPrepQuery2");
LOG.debug("Perpared query handle is " + handle);
@@ -135,29 +140,33 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
}
String status = qCom.getStatus(handle);
LOG.debug("Prepared Query Status is " + status);
- Assert.assertTrue(status.contains("Status : SUCCESSFUL"));
+ assertTrue(status.contains("Status : SUCCESSFUL"));
- result = qCom.getQueryResults(handle);
+ result = qCom.getQueryResults(handle, null);
+ LOG.debug("Prepared Query Result is " + result);
+ assertTrue(result.contains("1\tfirst"));
+ // Fetch again.
+ result = qCom.getQueryResults(handle, null);
LOG.debug("Prepared Query Result is " + result);
- Assert.assertTrue(result.contains("1\tfirst"));
+ assertTrue(result.contains("1\tfirst"));
result = qCom.destroyPreparedQuery(qh);
LOG.debug("destroy result is " + result);
- Assert.assertEquals("Successfully destroyed " + qh, result);
+ assertEquals("Successfully destroyed " + qh, result);
result = qCom.getAllPreparedQueries("testPreparedName", "all", submitTime, Long.MAX_VALUE);
- Assert.assertEquals(result, "No prepared queries");
+ assertEquals(result, "No prepared queries");
final String qh2 = qCom.explainAndPrepare(sql, "testPrepQuery3");
- Assert.assertTrue(qh2.contains(explainPlan));
+ assertTrue(qh2.contains(explainPlan));
String handles = qCom.getAllPreparedQueries("testPrepQuery3", "all", -1, Long.MAX_VALUE);
- Assert.assertFalse(handles.contains("No prepared queries"), handles);
+ assertFalse(handles.contains("No prepared queries"), handles);
String handles2 = qCom.getAllPreparedQueries("testPrepQuery3", "all", -1, submitTime - 1);
- Assert.assertFalse(handles2.contains(qh), handles2);
+ assertFalse(handles2.contains(qh), handles2);
result = qCom.destroyPreparedQuery(handles);
- Assert.assertEquals("Successfully destroyed " + handles, result);
+ assertEquals("Successfully destroyed " + handles, result);
}
/**
@@ -172,7 +181,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
qCom.setClient(client);
String sql = "cube select id, name from test_dim";
final String result = qCom.explainAndPrepare(sql, "testFailPrepared");
- Assert.assertTrue(result.contains("Explain FAILED:Error while processing statement: FAILED: Hive Internal Error:"
+ assertTrue(result.contains("Explain FAILED:Error while processing statement: FAILED: Hive Internal Error:"
+ " java.lang.ClassNotFoundException(org.apache.lens.driver.hive.TestHiveDriver.FailHook)"));
}
@@ -187,7 +196,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String result = qCom.explainQuery(sql, "");
LOG.debug(result);
- Assert.assertTrue(result.contains(explainPlan));
+ assertTrue(result.contains(explainPlan));
}
@@ -202,10 +211,10 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String result = qCom.explainQuery(sql, "");
LOG.debug(result);
- Assert.assertTrue(result.contains("Explain FAILED:"));
+ assertTrue(result.contains("Explain FAILED:"));
result = qCom.explainAndPrepare(sql, "");
- Assert.assertTrue(result.contains("Explain FAILED:"));
+ assertTrue(result.contains("Explain FAILED:"));
}
/**
@@ -222,61 +231,68 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
String user = qCom.getClient().getLensStatement(new QueryHandle(UUID.fromString(qh))).getQuery().getSubmittedUser();
String result = qCom.getAllQueries("", "testQuery1", user, -1, Long.MAX_VALUE);
// this is because previous query has run two query handle will be there
- Assert.assertTrue(result.contains(qh), result);
- Assert.assertTrue(result.contains("Total number of queries"));
+ assertTrue(result.contains(qh), result);
+ assertTrue(result.contains("Total number of queries"));
String[] resultSplits = result.split("\n");
// assert on the number of queries
- Assert.assertEquals(String.valueOf(resultSplits.length - 1), resultSplits[resultSplits.length - 1].split(": ")[1]);
+ assertEquals(String.valueOf(resultSplits.length - 1), resultSplits[resultSplits.length - 1].split(": ")[1]);
String details = qCom.getDetails(qh);
- Assert.assertTrue(details.contains("driverQuery"), details);
+ assertTrue(details.contains("driverQuery"), details);
// Check that query name searching is 'ilike'
String result2 = qCom.getAllQueries("", "query", "all", -1, Long.MAX_VALUE);
- Assert.assertTrue(result2.contains(qh), result2);
+ assertTrue(result2.contains(qh), result2);
while (!client.getQueryStatus(qh).finished()) {
Thread.sleep(5000);
}
- Assert.assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL"));
+ assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL"));
details = qCom.getDetails(qh);
- Assert.assertTrue(details.contains("driverQuery"));
+ assertTrue(details.contains("driverQuery"));
+
+ result = qCom.getQueryResults(qh, null);
+ assertTrue(result.contains("1\tfirst"));
+
+ downloadResult(qCom, qh, result);
+ // re-download should also succeed
+ downloadResult(qCom, qh, result);
- result = qCom.getQueryResults(qh);
- Assert.assertTrue(result.contains("1\tfirst"));
// Kill query is not tested as there is no deterministic way of killing a query
result = qCom.getAllQueries("SUCCESSFUL", "", "all", -1, Long.MAX_VALUE);
- Assert.assertTrue(result.contains(qh), result);
+ assertTrue(result.contains(qh), result);
result = qCom.getAllQueries("FAILED", "", "all", -1, Long.MAX_VALUE);
if (!result.contains("No queries")) {
// Make sure valid query handles are returned
String[] handles = StringUtils.split(result, "\n");
for (String handle : handles) {
- QueryHandle.fromString(handle.trim());
+ if (!handle.contains("Total number of queries")) {
+ QueryHandle.fromString(handle.trim());
+ }
}
}
String queryName = client.getLensStatement(new QueryHandle(UUID.fromString(qh))).getQuery().getQueryName();
- Assert.assertTrue("testQuery1".equalsIgnoreCase(queryName), queryName);
+ assertTrue("testQuery1".equalsIgnoreCase(queryName), queryName);
result = qCom.getAllQueries("", "", "", submitTime, System.currentTimeMillis());
- Assert.assertTrue(result.contains(qh), result);
+ assertTrue(result.contains(qh), result);
result = qCom.getAllQueries("", "fooBar", "all", submitTime, System.currentTimeMillis());
- Assert.assertTrue(result.contains("No queries"), result);
+ assertTrue(result.contains("No queries"), result);
result = qCom.getAllQueries("SUCCESSFUL", "", "all", submitTime, System.currentTimeMillis());
- Assert.assertTrue(result.contains(qh));
+ assertTrue(result.contains(qh));
result = qCom.getAllQueries("SUCCESSFUL", "", "all", submitTime - 5000, submitTime - 1);
// should not give query since its not in the range
- Assert.assertFalse(result.contains(qh));
+ assertFalse(result.contains(qh));
try {
// Should fail with bad request since fromDate > toDate
result = qCom.getAllQueries("SUCCESSFUL", "", "all", submitTime + 5000, submitTime);
- Assert.fail("Call should have failed with BadRequestException, instead got " + result);
+ fail("Call should have failed with BadRequestException, instead got " + result);
} catch (BadRequestException exc) {
// pass
}
@@ -284,6 +300,14 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
System.out.println("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@");
}
+ private void downloadResult(LensQueryCommands qCom, String qh, String expected) throws IOException {
+ assertTrue(qCom.getQueryResults(qh, resDir.getAbsolutePath()).contains("Saved"));
+ assertEquals(readFile(resDir.getAbsolutePath() + File.separator + qh + ".csv").trim(), expected.trim());
+ }
+ private String readFile(String path) throws FileNotFoundException {
+ return new Scanner(new File(path)).useDelimiter("\\Z").next();
+ }
+
/**
* Sets the up.
*
@@ -313,7 +337,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
timePart.getPartSpecElement().add(partElement);
xp.setTimePartitionSpec(timePart);
APIResult result = client.addPartitionToDim("dim_table", "local", xp);
- Assert.assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
+ assertEquals(result.getStatus(), APIResult.Status.SUCCEEDED);
}
/**
@@ -324,7 +348,7 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
private void testExecuteSyncQuery(LensQueryCommands qCom) {
String sql = "cube select id,name from test_dim";
String result = qCom.executeQuery(sql, false, "testQuery2");
- Assert.assertTrue(result.contains("1\tfirst"), result);
+ assertTrue(result.contains("1\tfirst"), result);
}
/**
@@ -340,11 +364,11 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
try {
String result = qCom.executeQuery(query, false, "testQuery3");
System.out.println("@@ RESULT " + result);
- Assert.assertNotNull(result);
- Assert.assertFalse(result.contains("Failed to get resultset"));
+ assertNotNull(result);
+ assertFalse(result.contains("Failed to get resultset"));
} catch (Exception exc) {
exc.printStackTrace();
- Assert.fail("Exception not expected: " + exc.getMessage());
+ fail("Exception not expected: " + exc.getMessage());
}
System.out.println("@@END_PERSISTENT_RESULT_TEST-------------");
}
@@ -364,17 +388,17 @@ public class TestLensQueryCommands extends LensCliApplicationTest {
while (!client.getQueryStatus(qh).finished()) {
Thread.sleep(5000);
}
- Assert.assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL"));
+ assertTrue(qCom.getStatus(qh).contains("Status : SUCCESSFUL"));
- String result = qCom.getQueryResults(qh);
+ String result = qCom.getQueryResults(qh, null);
System.out.println("@@ RESULT " + result);
- Assert.assertNotNull(result);
+ assertNotNull(result);
// This is to check for positive processing time
- Assert.assertFalse(result.contains("(-"));
+ assertFalse(result.contains("(-"));
} catch (Exception exc) {
exc.printStackTrace();
- Assert.fail("Exception not expected: " + exc.getMessage());
+ fail("Exception not expected: " + exc.getMessage());
}
System.out.println("@@END_FINISHED_PURGED_RESULT_TEST-------------");
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java b/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java
index e5fe067..1910be9 100644
--- a/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java
+++ b/lens-cli/src/test/java/org/apache/lens/cli/doc/TestGenerateCLIUserDoc.java
@@ -21,10 +21,7 @@ package org.apache.lens.cli.doc;
import java.io.*;
import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
-import java.util.ArrayList;
-import java.util.Comparator;
-import java.util.HashSet;
-import java.util.TreeSet;
+import java.util.*;
import org.apache.lens.cli.commands.*;
import org.apache.lens.cli.commands.annotations.UserDocumentation;
@@ -48,7 +45,7 @@ public class TestGenerateCLIUserDoc {
BufferedWriter bw = new BufferedWriter(new FileWriter(new File(APT_FILE)));
StringBuilder sb = new StringBuilder();
sb.append(getCLIIntroduction()).append("\n\n\n");
- ArrayList<Class<? extends BaseLensCommand>> classes = Lists.newArrayList(
+ List<Class<? extends BaseLensCommand>> classes = Lists.newArrayList(
LensConnectionCommands.class,
LensDatabaseCommands.class,
LensStorageCommands.class,
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-client/src/main/java/org/apache/lens/client/LensClient.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensClient.java b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
index 9b0c935..cd8bf16 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensClient.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensClient.java
@@ -22,6 +22,8 @@ import java.util.Date;
import java.util.HashMap;
import java.util.List;
+import javax.ws.rs.core.Response;
+
import org.apache.lens.api.APIResult;
import org.apache.lens.api.metastore.*;
import org.apache.lens.api.query.*;
@@ -30,15 +32,18 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import com.google.common.collect.Maps;
+import lombok.Getter;
public class LensClient {
private static final Log LOG = LogFactory.getLog(LensClient.class);
private static final String DEFAULT_PASSWORD = "";
private final LensClientConfig conf;
+ @Getter
private final LensMetadataClient mc;
private String password;
- private LensConnection conn;
+ @Getter
+ private LensConnection connection;
private final HashMap<QueryHandle, LensStatement> statementMap =
Maps.newHashMap();
private final LensStatement statement;
@@ -63,8 +68,8 @@ public class LensClient {
this.conf.set(LensClientConfig.SESSION_CLUSTER_USER, System.getProperty("user.name"));
}
connectToLensServer();
- mc = new LensMetadataClient(conn);
- statement = new LensStatement(conn);
+ mc = new LensMetadataClient(connection);
+ statement = new LensStatement(connection);
}
public LensClient(Credentials cred) {
@@ -80,10 +85,6 @@ public class LensClient {
return query.getQueryHandle();
}
- public LensConnection getConnection() {
- return conn;
- }
-
public Date getLatestDateOfCube(String cubeName, String timePartition) {
return mc.getLatestDateOfCube(cubeName, timePartition);
}
@@ -150,16 +151,24 @@ public class LensClient {
return getResultsFromHandle(q);
}
+ public Response getHttpResults() {
+ return statement.getHttpResultSet();
+ }
+
+ public Response getHttpResults(QueryHandle q) {
+ return statement.getHttpResultSet(statement.getQuery(q));
+ }
+
public LensStatement getLensStatement(QueryHandle query) {
return this.statementMap.get(query);
}
public QueryStatus getQueryStatus(QueryHandle query) {
- return new LensStatement(conn).getQuery(query).getStatus();
+ return new LensStatement(connection).getQuery(query).getStatus();
}
public LensQuery getQueryDetails(QueryHandle handle) {
- return new LensStatement(conn).getQuery(handle);
+ return new LensStatement(connection).getQuery(handle);
}
public QueryStatus getQueryStatus(String q) {
@@ -171,7 +180,7 @@ public class LensClient {
}
public QueryPlan getQueryPlan(String q) {
- return new LensStatement(conn).explainQuery(q);
+ return new LensStatement(connection).explainQuery(q);
}
public boolean killQuery(QueryHandle q) {
@@ -190,15 +199,15 @@ public class LensClient {
}
public List<QueryHandle> getQueries(String state, String queryName, String user, long fromDate, long toDate) {
- return new LensStatement(conn).getAllQueries(state, queryName, user, fromDate, toDate);
+ return new LensStatement(connection).getAllQueries(state, queryName, user, fromDate, toDate);
}
private void connectToLensServer() {
LOG.debug("Connecting to lens server " + new LensConnectionParams(conf));
- conn = new LensConnection(new LensConnectionParams(conf));
- conn.open(password);
- LOG.debug("Successfully connected to server " + conn);
+ connection = new LensConnection(new LensConnectionParams(conf));
+ connection.open(password);
+ LOG.debug("Successfully connected to server " + connection);
}
@@ -269,36 +278,36 @@ public class LensClient {
}
public APIResult setConnectionParam(String key, String val) {
- return this.conn.setConnectionParams(key, val);
+ return this.connection.setConnectionParams(key, val);
}
public List<String> getConnectionParam() {
- return this.conn.getConnectionParams();
+ return this.connection.getConnectionParams();
}
public List<String> getConnectionParam(String key) {
- return this.conn.getConnectionParams(key);
+ return this.connection.getConnectionParams(key);
}
public APIResult closeConnection() {
LOG.debug("Closing lens connection: " + new LensConnectionParams(conf));
- return this.conn.close();
+ return this.connection.close();
}
public APIResult addJarResource(String path) {
- return this.conn.addResourceToConnection("jar", path);
+ return this.connection.addResourceToConnection("jar", path);
}
public APIResult removeJarResource(String path) {
- return this.conn.removeResourceFromConnection("jar", path);
+ return this.connection.removeResourceFromConnection("jar", path);
}
public APIResult addFileResource(String path) {
- return this.conn.addResourceToConnection("file", path);
+ return this.connection.addResourceToConnection("file", path);
}
public APIResult removeFileResource(String path) {
- return this.conn.removeResourceFromConnection("file", path);
+ return this.connection.removeResourceFromConnection("file", path);
}
public APIResult createFactTable(String factSpec) {
@@ -523,11 +532,11 @@ public class LensClient {
}
public boolean isConnectionOpen() {
- return this.conn.isOpen();
+ return this.connection.isOpen();
}
public List<String> listResources(String type) {
- return this.conn.listResourcesFromConnection(type);
+ return this.connection.listResourcesFromConnection(type);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
----------------------------------------------------------------------
diff --git a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
index 5ce3a82..4472a80 100644
--- a/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
+++ b/lens-client/src/main/java/org/apache/lens/client/LensStatement.java
@@ -26,6 +26,7 @@ import javax.ws.rs.client.Entity;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
import org.apache.lens.api.APIResult;
import org.apache.lens.api.query.*;
@@ -268,7 +269,7 @@ public class LensStatement {
}
Client client = ClientBuilder.newBuilder().register(MultiPartFeature.class).register(LensJAXBContextResolver.class)
- .build();
+ .build();
FormDataMultiPart mp = new FormDataMultiPart();
mp.bodyPart(new FormDataBodyPart(FormDataContentDisposition.name("sessionid").build(), connection
@@ -281,7 +282,7 @@ public class LensStatement {
WebTarget target = getQueryWebTarget(client);
return target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryHandle, NoErrorPayload>>(){}).getData();
+ new GenericType<LensResponse<QueryHandle, NoErrorPayload>>() {}).getData();
}
/**
@@ -332,7 +333,7 @@ public class LensStatement {
WebTarget target = getQueryWebTarget(client);
QueryPlan handle = target.request().post(Entity.entity(mp, MediaType.MULTIPART_FORM_DATA_TYPE),
- new GenericType<LensResponse<QueryPlan, NoErrorPayload>>() {}).getData();
+ new GenericType<LensResponse<QueryPlan, NoErrorPayload>>() {}).getData();
return handle;
}
@@ -405,6 +406,10 @@ public class LensStatement {
return this.getResultSet(this.query);
}
+ public Response getHttpResultSet() {
+ return this.getHttpResultSet(this.query);
+ }
+
/**
* Gets the result set.
*
@@ -427,6 +432,27 @@ public class LensStatement {
}
/**
+ * Gets the http result set.
+ *
+ * @param query the query
+ * @return the http result set
+ */
+ public Response getHttpResultSet(LensQuery query) {
+ if (query.getStatus().getStatus() != QueryStatus.Status.SUCCESSFUL) {
+ throw new IllegalArgumentException("Result set metadata " + "can be only queries for successful queries");
+ }
+ Client client = ClientBuilder.newClient();
+
+ try {
+ WebTarget target = getQueryWebTarget(client);
+ return target.path(query.getQueryHandle().toString()).path("httpresultset")
+ .queryParam("sessionid", connection.getSessionHandle()).request().get();
+ } catch (Exception e) {
+ throw new IllegalStateException("Failed to get resultset, cause:" + e.getMessage());
+ }
+ }
+
+ /**
* Kill.
*
* @return true, if successful
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
index 43985ba..8bec5e0 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveDriver.java
@@ -801,12 +801,10 @@ public class HiveDriver implements LensDriver {
OperationHandle op = getHiveHandle(context.getQueryHandle());
LOG.info("Creating result set for hiveHandle:" + op);
try {
- if (op.hasResultSet() || context.isDriverPersistent()) {
- if (context.isDriverPersistent()) {
- return new HivePersistentResultSet(new Path(context.getHdfsoutPath()), op, getClient());
- } else {
- return new HiveInMemoryResultSet(op, getClient(), closeAfterFetch);
- }
+ if (context.isDriverPersistent()) {
+ return new HivePersistentResultSet(new Path(context.getHdfsoutPath()), op, getClient());
+ } else if (op.hasResultSet()) {
+ return new HiveInMemoryResultSet(op, getClient(), closeAfterFetch);
} else {
// queries that do not have result
return null;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
index a953ec6..4d631ec 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
@@ -61,6 +61,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
/** The num columns. */
int numColumns;
+ private FetchOrientation orientation;
/**
* Instantiates a new hive in memory result set.
@@ -77,6 +78,7 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
this.closeAfterFecth = closeAfterFecth;
this.metadata = client.getResultSetMetadata(opHandle);
this.numColumns = metadata.getColumnDescriptors().size();
+ this.seekToStart();
}
/*
@@ -98,16 +100,23 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
return hrsMeta;
}
+ @Override
+ public boolean seekToStart() {
+ orientation = FetchOrientation.FETCH_FIRST;
+ return true;
+ }
+
/*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext()
- */
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext()
+ */
@Override
public boolean hasNext() throws LensException {
if (fetchedRowsItr == null || !fetchedRowsItr.hasNext()) {
try {
- rowSet = client.fetchResults(opHandle, FetchOrientation.FETCH_NEXT, fetchSize);
+ rowSet = client.fetchResults(opHandle, orientation, fetchSize);
+ orientation = FetchOrientation.FETCH_NEXT;
noMoreResults = rowSet.numRows() == 0;
if (noMoreResults) {
if (closeAfterFecth) {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
index 36a48b9..bbcc3f1 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
@@ -76,10 +76,11 @@ public class JDBCResultSet extends InMemoryResultSet {
* @param resultSet the result set
* @param closeAfterFetch the close after fetch
*/
- public JDBCResultSet(QueryResult queryResult, ResultSet resultSet, boolean closeAfterFetch) {
+ public JDBCResultSet(QueryResult queryResult, ResultSet resultSet, boolean closeAfterFetch) throws LensException {
this.queryResult = queryResult;
this.resultSet = resultSet;
this.closeAfterFetch = closeAfterFetch;
+ seekToStart();
}
private ResultSetMetaData getRsMetadata() throws LensException {
@@ -294,11 +295,23 @@ public class JDBCResultSet extends InMemoryResultSet {
}
}
+ @Override
+ public boolean seekToStart() throws LensException {
+ try {
+ if (!resultSet.isBeforeFirst()) {
+ resultSet.beforeFirst();
+ }
+ return true;
+ } catch (SQLException e) {
+ throw new LensException(e);
+ }
+ }
+
/*
- * (non-Javadoc)
- *
- * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext()
- */
+ * (non-Javadoc)
+ *
+ * @see org.apache.lens.server.api.driver.InMemoryResultSet#hasNext()
+ */
@Override
public synchronized boolean hasNext() throws LensException {
try {
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java
index f11d883..3b76126 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/InMemoryResultSet.java
@@ -31,6 +31,7 @@ import org.apache.lens.server.api.error.LensException;
*/
public abstract class InMemoryResultSet extends LensResultSet {
+ public abstract boolean seekToStart() throws LensException;
/**
* Whether there is another result row available.
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 0f4e990..30c1a71 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -308,6 +308,11 @@ public class MockDriver implements LensDriver {
}
@Override
+ public boolean seekToStart() throws LensException {
+ return false;
+ }
+
+ @Override
public boolean hasNext() throws LensException {
// TODO Auto-generated method stub
return false;
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index 343cd01..4cf1fa9 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1288,6 +1288,9 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
}
}
}
+ if (resultSets.get(queryHandle) instanceof InMemoryResultSet) {
+ ((InMemoryResultSet) resultSets.get(queryHandle)).seekToStart();
+ }
return resultSets.get(queryHandle);
}
}
@@ -2288,7 +2291,12 @@ public class QueryExecutionServiceImpl extends LensService implements QueryExecu
.type(MediaType.APPLICATION_OCTET_STREAM).build();
}
} else {
- throw new NotFoundException("Http result not available for query:" + queryHandle.toString());
+ String entity = "";
+ if (result instanceof InMemoryResultSet || result instanceof PersistentResultSet) {
+ entity = "Result is available in driver's "
+ + (result instanceof InMemoryResultSet ? "memory" : "persistence") + ".";
+ }
+ return Response.status(Response.Status.NOT_FOUND).entity(entity).build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/d47af610/src/site/apt/user/cli.apt
----------------------------------------------------------------------
diff --git a/src/site/apt/user/cli.apt b/src/site/apt/user/cli.apt
index 8efb58b..2016bfd 100644
--- a/src/site/apt/user/cli.apt
+++ b/src/site/apt/user/cli.apt
@@ -324,7 +324,7 @@ User CLI Commands
*--+--+
|query list [--state \<query-status\>] [--name \<query-name\>] [--user \<user-who-submitted-query\>] [--fromDate \<submission-time-is-after\>] [--toDate \<submission-time-is-before\>]|Get all queries. Various filter options can be provided(optionally), as can be seen from the command syntax|
*--+--+
-|query results [--query_handle] \<query_handle\>|get results of async query with query handle <<<query_handle>>>|
+|query results [--query_handle] \<query_handle\> [--save_location \<save_location\>]|get results of async query with query handle <<<query_handle>>>. Can optionally save the results to a file by providing <<<save_location>>>|
*--+--+
|query status [--query_handle] \<query_handle\>|Fetch status of executed query having query handle <<<query_handle>>>|
*--+--+