You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2017/07/13 09:14:03 UTC
[10/12] lens git commit: LENS-1449 : lens.query.result.parent.dir to
be set at driver level.
LENS-1449 : lens.query.result.parent.dir to be set at driver level.
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/41051ea4
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/41051ea4
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/41051ea4
Branch: refs/heads/current-release-line
Commit: 41051ea47f164f004f5fccaa7ea91765007811fc
Parents: 503a46d
Author: Raghavendra Singh <ra...@apache.org>
Authored: Thu Jul 6 14:03:59 2017 +0530
Committer: Rajat Khandelwal <ra...@gmail.com>
Committed: Thu Jul 13 14:42:59 2017 +0530
----------------------------------------------------------------------
.../api/query/DriverSelectorQueryContext.java | 4 +++-
.../lens/server/api/query/QueryContext.java | 5 +++++
.../lens/server/query/TestQueryService.java | 6 ++++--
.../lens/server/query/TestResultFormatting.java | 20 +++++++++++++++++++-
.../drivers/hive/hive1/hivedriver-site.xml | 5 +++++
.../drivers/hive/hive2/hivedriver-site.xml | 6 ++++++
.../drivers/jdbc/jdbc1/jdbcdriver-site.xml | 5 +++++
lens-server/src/test/resources/lens-site.xml | 5 -----
8 files changed, 47 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
index 8e431d1..41e533e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DriverSelectorQueryContext.java
@@ -155,7 +155,9 @@ public class DriverSelectorQueryContext {
private Configuration mergeConf(LensDriver driver, Configuration queryConf) {
Configuration conf = new Configuration(driver.getConf());
for (Map.Entry<String, String> entry : queryConf) {
- conf.set(entry.getKey(), entry.getValue());
+ if (!conf.getFinalParameters().contains(entry.getKey())) {
+ conf.set(entry.getKey(), entry.getValue());
+ }
}
conf.setClassLoader(queryConf.getClassLoader());
return conf;
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 9923589..ccdef87 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -328,6 +328,11 @@ public class QueryContext extends AbstractQueryContext implements FailureContext
}
public String getResultSetParentDir() {
+ if (getSelectedDriver() != null && getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) != null) {
+ log.info("Fetching Parent Dir from driver conf:- "
+ + getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR));
+ return getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR);
+ }
return conf.get(LensConfConstants.RESULT_SET_PARENT_DIR, LensConfConstants.RESULT_SET_PARENT_DIR_DEFAULT);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index 1149696..b7cdb88 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -848,9 +848,11 @@ public class TestQueryService extends LensJerseyTest {
LensResultSet rs = queryService.getResultset(handle3);
//check persisted result path
String expectedPath =
- ctx3.getConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + handle3.getHandleIdString()
+ ctx3.getSelectedDriverConf().get(LensConfConstants.RESULT_SET_PARENT_DIR) + "/" + handle3.getHandleIdString()
+ ctx3.getConf().get(LensConfConstants.QUERY_OUTPUT_FILE_EXTN);
- assertTrue(((PersistentResultSet) rs).getOutputPath().endsWith(expectedPath));
+ assertTrue(((PersistentResultSet) rs).getOutputPath().endsWith(expectedPath)
+ , "Result Path " + ((PersistentResultSet) rs).getOutputPath()
+ + " does not contain expected path: " + expectedPath);
validateHttpEndPoint(target(), null, handle3, null);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
index 4f8b8ff..76d3720 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestResultFormatting.java
@@ -157,6 +157,22 @@ public class TestResultFormatting extends LensJerseyTest {
}
/**
+ * Test result formatter hdfs persistent result location override from driver conf.
+ *
+ * @throws InterruptedException the interrupted exception
+ * @throws IOException Signals that an I/O exception has occurred.
+ */
+ @Test
+ public void testResultDirOverrideFromDriverConf() throws InterruptedException, IOException {
+ LensConf conf = new LensConf();
+ conf.addProperty(LensConfConstants.QUERY_PERSISTENT_RESULT_INDRIVER, "true");
+ LensQuery lensQuery = testResultFormatter(conf, QueryStatus.Status.SUCCESSFUL, false,
+ null, MediaType.APPLICATION_XML_TYPE);
+ assertEquals(lensQuery.getResultSetPath(), "file:"+System.getProperty("user.dir")+"/target/hive-lens-results/"
+ +lensQuery.getQueryHandle().getHandleIdString()+".csv");
+ }
+
+ /**
* Test persistent result with max size.
*
* @throws InterruptedException the interrupted exception
@@ -196,7 +212,7 @@ public class TestResultFormatting extends LensJerseyTest {
* @throws InterruptedException the interrupted exception
* @throws IOException Signals that an I/O exception has occurred.
*/
- private void testResultFormatter(LensConf conf, Status status, boolean isDir, String reDirectUrl, MediaType mt)
+ private LensQuery testResultFormatter(LensConf conf, Status status, boolean isDir, String reDirectUrl, MediaType mt)
throws InterruptedException, IOException {
// test post execute op
final WebTarget target = target().path("queryapi/queries");
@@ -266,6 +282,8 @@ public class TestResultFormatting extends LensJerseyTest {
|| ctx.getStatus().getStatusMessage().equals(ResultFormatter.ERROR_MESSAGE));
assertEquals(ctx.getStatus().getErrorMessage(), "Class NonexistentSerde.class not found");
}
+
+ return ctx;
}
@AfterTest
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
index 1f0ff43..83143e7 100644
--- a/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
+++ b/lens-server/src/test/resources/drivers/hive/hive1/hivedriver-site.xml
@@ -82,4 +82,9 @@
</description>
</property>
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/hive-lens-results</value>
+ <final>true</final>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
index eb902e0..4170e9b 100644
--- a/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
+++ b/lens-server/src/test/resources/drivers/hive/hive2/hivedriver-site.xml
@@ -82,4 +82,10 @@
</description>
</property>
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/hive-lens-results</value>
+ <final>true</final>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml b/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
index 5b8b43f..ad068f8 100644
--- a/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
+++ b/lens-server/src/test/resources/drivers/jdbc/jdbc1/jdbcdriver-site.xml
@@ -55,4 +55,9 @@
<name>lens.driver.test.drivername</name>
<value>jdbc/jdbc1</value>
</property>
+ <property>
+ <name>lens.query.result.parent.dir</name>
+ <value>target/jdbc-lens-results</value>
+ <final>true</final>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/lens/blob/41051ea4/lens-server/src/test/resources/lens-site.xml
----------------------------------------------------------------------
diff --git a/lens-server/src/test/resources/lens-site.xml b/lens-server/src/test/resources/lens-site.xml
index 0060fa7..7e5f522 100644
--- a/lens-server/src/test/resources/lens-site.xml
+++ b/lens-server/src/test/resources/lens-site.xml
@@ -49,11 +49,6 @@
<name>lens.server.persist.location</name>
<value>target/persist-dir</value>
</property>
-
- <property>
- <name>lens.query.result.parent.dir</name>
- <value>target/lens-results</value>
- </property>
<property>
<name>hive.server2.authentication</name>