You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by ra...@apache.org on 2018/06/22 11:47:28 UTC
lens git commit: LENS-1521 : Download url config needs to be specific
to Query and not Service
Repository: lens
Updated Branches:
refs/heads/master 11cc1859f -> 3076b26c8
LENS-1521 : Download url config needs to be specific to Query and not Service
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/3076b26c
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/3076b26c
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/3076b26c
Branch: refs/heads/master
Commit: 3076b26c81d3d0d3511e9ee6bf00033a5d22149c
Parents: 11cc185
Author: Amareshwari Sriramadasu <am...@apache.org>
Authored: Fri Jun 22 17:16:15 2018 +0530
Committer: Rajitha.R <ra...@IM0318-L0.corp.inmobi.com>
Committed: Fri Jun 22 17:16:15 2018 +0530
----------------------------------------------------------------------
.../main/java/org/apache/lens/api/LensConf.java | 4 ++++
.../query/DefaultDownloadResultUrlProvider.java | 5 +++--
.../api/query/DownloadResultUrlProvider.java | 10 +++++++++-
.../lens/server/api/query/QueryContext.java | 1 -
.../lens/server/query/LensPersistentResult.java | 19 +++++++++++--------
.../server/query/QueryExecutionServiceImpl.java | 12 +++++-------
6 files changed, 32 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-api/src/main/java/org/apache/lens/api/LensConf.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/LensConf.java b/lens-api/src/main/java/org/apache/lens/api/LensConf.java
index 86504c7..d48e4c2 100644
--- a/lens-api/src/main/java/org/apache/lens/api/LensConf.java
+++ b/lens-api/src/main/java/org/apache/lens/api/LensConf.java
@@ -79,4 +79,8 @@ public class LensConf extends ToYAMLString implements Serializable {
public String getProperty(Object key) {
return properties.get(key);
}
+
+ public String getProperty(Object key, String defaultValue) {
+ return properties.getOrDefault(key, defaultValue);
+ }
}
http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java
index cf8a066..786a23e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DefaultDownloadResultUrlProvider.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.server.api.query;
+import org.apache.lens.api.LensConf;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.hadoop.conf.Configuration;
@@ -28,8 +29,8 @@ import lombok.extern.slf4j.Slf4j;
public class DefaultDownloadResultUrlProvider implements DownloadResultUrlProvider {
@Override
- public String getResultUrl(Configuration conf, String queryHandle) {
- log.debug("Returning Default result set url ");
+ public String getResultUrl(Configuration conf, LensConf qconf, String queryHandle) {
+ log.debug("Returning Default result set url " + conf.get(LensConfConstants.SERVER_BASE_URL));
return conf.get(LensConfConstants.SERVER_BASE_URL, LensConfConstants.DEFAULT_SERVER_BASE_URL)
+ "queryapi/queries/" + queryHandle + "/httpresultset";
}
http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java
index 136bf60..55fd127 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/DownloadResultUrlProvider.java
@@ -18,6 +18,8 @@
*/
package org.apache.lens.server.api.query;
+import org.apache.lens.api.LensConf;
+
import org.apache.hadoop.conf.Configuration;
/*
@@ -25,5 +27,11 @@ import org.apache.hadoop.conf.Configuration;
* */
public interface DownloadResultUrlProvider {
- String getResultUrl(Configuration conf, String queryHandle);
+ /*
+ @param : conf : server conf
+ @param : qconf : query conf
+ @param : queryHandle : the query handle
+ @return : download url sent to user via email
+ * */
+ String getResultUrl(Configuration conf, LensConf qconf, String queryHandle);
}
http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
index 8176f03..a0d8e0b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryContext.java
@@ -280,7 +280,6 @@ public class QueryContext extends AbstractQueryContext implements FailureContext
if (selectedDriver != null) {
this.setSelectedDriver(selectedDriver);
}
- this.lensConf = qconf;
this.driverStatus = new DriverQueryStatus();
}
http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
index 4aa4578..443ad9d 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
@@ -20,6 +20,7 @@ package org.apache.lens.server.query;
import java.io.IOException;
+import org.apache.lens.api.LensConf;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.server.api.LensConfConstants;
import org.apache.lens.server.api.driver.LensResultSetMetadata;
@@ -32,7 +33,7 @@ import org.apache.lens.server.api.query.QueryContext;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.util.ReflectionUtils;
+import org.apache.hadoop.util.ReflectionUtils;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
@@ -53,6 +54,7 @@ public class LensPersistentResult extends PersistentResultSet {
/** The file size. */
private final Long fileSize;
private final Configuration conf;
+ private final LensConf queryConf;
@Getter
private String httpResultUrl = null;
@@ -68,35 +70,36 @@ public class LensPersistentResult extends PersistentResultSet {
*/
public LensPersistentResult(QueryHandle queryHandle, LensResultSetMetadata metadata, String outputPath, Integer
numRows, Long fileSize,
- Configuration conf) {
+ Configuration conf, LensConf qconf) {
this.metadata = metadata;
this.outputPath = outputPath;
this.numRows = numRows;
this.fileSize = fileSize;
this.conf = conf;
+ this.queryConf = qconf;
if (isHttpResultAvailable()) {
resultUrlSetter = ReflectionUtils.newInstance(this.conf.getClass(
LensConfConstants.RESULT_DOWNLOAD_URL_PROVIDER_CLASS,
- LensConfConstants.DEFAULT_RESULT_DOWNLOAD_URL_PROVIDER, DownloadResultUrlProvider.class));
- this.httpResultUrl = resultUrlSetter.getResultUrl(this.conf, queryHandle.toString());
+ LensConfConstants.DEFAULT_RESULT_DOWNLOAD_URL_PROVIDER, DownloadResultUrlProvider.class), this.conf);
+ this.httpResultUrl = resultUrlSetter.getResultUrl(this.conf, this.queryConf, queryHandle.toString());
log.info("Config : " + this.conf.get(LensConfConstants.RESULT_DOWNLOAD_URL_PROVIDER_CLASS)
+ " Result url set as : " + this.httpResultUrl);
}
}
- public LensPersistentResult(QueryContext ctx, Configuration conf) {
+ LensPersistentResult(QueryContext ctx, Configuration conf, LensConf qconf) {
this(ctx.getQueryHandle(),
ctx.getQueryOutputFormatter().getMetadata(),
ctx.getQueryOutputFormatter().getFinalOutputPath(),
ctx.getQueryOutputFormatter().getNumRows(),
- ctx.getQueryOutputFormatter().getFileSize(), conf);
+ ctx.getQueryOutputFormatter().getFileSize(), conf, qconf);
}
- public LensPersistentResult(FinishedLensQuery query, Configuration conf) throws
+ LensPersistentResult(FinishedLensQuery query, Configuration conf, LensConf qconf) throws
ClassNotFoundException, IOException {
this(QueryHandle.fromString(query.getHandle()),
LensResultSetMetadata.fromJson(query.getMetadata()),
- query.getResult(), query.getRows(), query.getFileSize(), conf);
+ query.getResult(), query.getRows(), query.getFileSize(), conf, qconf);
}
@Override
http://git-wip-us.apache.org/repos/asf/lens/blob/3076b26c/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index d925b79..3f1731e 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -1938,11 +1938,11 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
throw new NotFoundException("InMemory Query result purged " + queryHandle);
}
try {
- Configuration queryConf = conf;
- if (ctx != null && ctx.getConf() != null) {
- queryConf.addResource(ctx.getConf());
+ LensConf qConf = null;
+ if (ctx != null && ctx.getLensConf() != null) {
+ qConf = ctx.getLensConf();
}
- return new LensPersistentResult(query, queryConf);
+ return new LensPersistentResult(query, conf, qConf);
} catch (Exception e) {
throw new LensException(e);
}
@@ -1971,9 +1971,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
LensResultSet resultSet = resultSets.get(queryHandle);
if (resultSet == null) {
if (ctx.isPersistent() && ctx.getQueryOutputFormatter() != null) {
- Configuration queryConf = conf;
- queryConf.addResource(ctx.getConf());
- resultSets.put(queryHandle, new LensPersistentResult(ctx, queryConf));
+ resultSets.put(queryHandle, new LensPersistentResult(ctx, conf, ctx.getLensConf()));
} else if (ctx.isResultAvailableInDriver() && !ctx.isQueryClosedOnDriver()) {
//InMemory result can not be returned for a closed query
resultSet = getDriverResultset(queryHandle);