You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by am...@apache.org on 2016/09/20 05:29:48 UTC
lens git commit: LENS-1317 : Lens Query Conf to be persisted in
Finished_Queries store
Repository: lens
Updated Branches:
refs/heads/master fe4ea6bd6 -> 4fa61d994
LENS-1317 : Lens Query Conf to be persisted in Finished_Queries store
Project: http://git-wip-us.apache.org/repos/asf/lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/lens/commit/4fa61d99
Tree: http://git-wip-us.apache.org/repos/asf/lens/tree/4fa61d99
Diff: http://git-wip-us.apache.org/repos/asf/lens/diff/4fa61d99
Branch: refs/heads/master
Commit: 4fa61d9940af6ede89401d5837f1f7796810f254
Parents: fe4ea6b
Author: Srikanth Sundarrajan <sr...@apache.org>
Authored: Tue Sep 20 10:59:28 2016 +0530
Committer: Amareshwari Sriramadasu <am...@apache.org>
Committed: Tue Sep 20 10:59:28 2016 +0530
----------------------------------------------------------------------
.../server/api/query/FinishedLensQuery.java | 9 +++--
.../apache/lens/server/query/LensServerDAO.java | 38 +++++++++++++++++---
.../apache/lens/server/query/TestLensDAO.java | 5 +++
3 files changed, 45 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index edd4284..b58fcf9 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -46,7 +46,7 @@ import lombok.ToString;
*
* @see java.lang.Object#hashCode()
*/
-@EqualsAndHashCode(exclude = "selectedDriver")
+@EqualsAndHashCode(exclude = {"selectedDriver", "conf"})
/*
* (non-Javadoc)
*
@@ -174,6 +174,10 @@ public class FinishedLensQuery {
@Setter
private String priority;
+ @Getter
+ @Setter
+ private LensConf conf;
+
/**
* The selected driver's query.
*/
@@ -217,6 +221,7 @@ public class FinishedLensQuery {
if (null != ctx.getPriority()) {
this.priority = ctx.getPriority().toString();
}
+ this.conf = ctx.getLensConf();
}
public QueryContext toQueryContext(Configuration conf, Collection<LensDriver> drivers) {
@@ -226,7 +231,7 @@ public class FinishedLensQuery {
}
QueryContext qctx =
- new QueryContext(userQuery, submitter, new LensConf(), conf, drivers, selectedDriver, submissionTime,
+ new QueryContext(userQuery, submitter, this.conf, conf, drivers, selectedDriver, submissionTime,
false);
qctx.setQueryHandle(QueryHandle.fromString(handle));
http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index 2ec938e..997b5a8 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -18,6 +18,7 @@
*/
package org.apache.lens.server.query;
+import java.nio.charset.Charset;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
@@ -26,12 +27,16 @@ import java.util.List;
import javax.sql.DataSource;
+import org.apache.lens.api.LensConf;
import org.apache.lens.api.query.QueryHandle;
import org.apache.lens.api.query.QueryStatus;
import org.apache.lens.server.api.error.LensException;
import org.apache.lens.server.api.query.FinishedLensQuery;
import org.apache.lens.server.util.UtilityMethods;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.dbutils.BasicRowProcessor;
+import org.apache.commons.dbutils.BeanProcessor;
import org.apache.commons.dbutils.DbUtils;
import org.apache.commons.dbutils.QueryRunner;
import org.apache.commons.dbutils.ResultSetHandler;
@@ -87,7 +92,8 @@ public class LensServerDAO {
+ "starttime bigint, " + "endtime bigint," + "result varchar(255)," + "status varchar(255), "
+ "metadata varchar(100000), " + "rows int, " + "filesize bigint, " + "errormessage varchar(10000), "
+ "driverstarttime bigint, " + "driverendtime bigint, " + "drivername varchar(10000), "
- + "queryname varchar(255), " + "submissiontime bigint, " + "driverquery varchar(1000000)" + ")";
+ + "queryname varchar(255), " + "submissiontime bigint, " + "driverquery varchar(1000000), "
+ + "conf varchar(100000))";
try {
QueryRunner runner = new QueryRunner(ds);
runner.update(sql);
@@ -110,8 +116,8 @@ public class LensServerDAO {
Connection conn = null;
String sql = "insert into finished_queries (handle, userquery, submitter, priority, "
+ "starttime,endtime,result,status,metadata,rows,filesize,"
- + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime, driverquery)"
- + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+ + "errormessage,driverstarttime,driverendtime, drivername, queryname, submissiontime, driverquery, conf)"
+ + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
try {
conn = getConnection();
QueryRunner runner = new QueryRunner();
@@ -119,7 +125,7 @@ public class LensServerDAO {
query.getStartTime(), query.getEndTime(), query.getResult(), query.getStatus(), query.getMetadata(),
query.getRows(), query.getFileSize(), query.getErrorMessage(), query.getDriverStartTime(),
query.getDriverEndTime(), query.getDriverName(), query.getQueryName(), query.getSubmissionTime(),
- query.getDriverQuery());
+ query.getDriverQuery(), serializeConf(query.getConf()));
conn.commit();
} finally {
DbUtils.closeQuietly(conn);
@@ -137,6 +143,15 @@ public class LensServerDAO {
}
}
+ private String serializeConf(LensConf conf) {
+ return Base64.encodeBase64String(conf.toXMLString().getBytes(Charset.defaultCharset()));
+ }
+
+ private LensConf deserializeConf(String serializedConf) {
+ return LensConf.fromXMLString(new String(Base64.decodeBase64(serializedConf),
+ Charset.defaultCharset()), LensConf.class);
+ }
+
/**
* Fetch Finished query from Database.
*
@@ -144,7 +159,8 @@ public class LensServerDAO {
* @return Finished query.
*/
public FinishedLensQuery getQuery(String handle) {
- ResultSetHandler<FinishedLensQuery> rsh = new BeanHandler<FinishedLensQuery>(FinishedLensQuery.class);
+ ResultSetHandler<FinishedLensQuery> rsh = new BeanHandler<>(FinishedLensQuery.class,
+ new BasicRowProcessor(new FinishedLensQueryBeanProcessor()));
String sql = "select * from finished_queries where handle=?";
QueryRunner runner = new QueryRunner(ds);
try {
@@ -155,6 +171,18 @@ public class LensServerDAO {
return null;
}
+ private class FinishedLensQueryBeanProcessor extends BeanProcessor {
+
+ @Override
+ protected Object processColumn(ResultSet rs, int index, Class<?> propType) throws SQLException {
+ Object obj = super.processColumn(rs, index, propType);
+ if (obj != null && propType.equals(LensConf.class) && obj instanceof String) {
+ return deserializeConf((String) obj);
+ }
+ return obj;
+ }
+ }
+
/**
* Find finished queries.
*
http://git-wip-us.apache.org/repos/asf/lens/blob/4fa61d99/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
index 2a820ec..e620dc5 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestLensDAO.java
@@ -72,6 +72,8 @@ public class TestLensDAO {
new Configuration(), 0);
long submissionTime = queryContext.getSubmissionTime();
queryContext.setQueryName("daoTestQuery1");
+ queryContext.getDriverContext().setSelectedDriver(new MockDriver());
+ queryContext.getLensConf().addProperty("prop", "value");
LensDriver mockDriver = new MockDriver();
DriverSelectorQueryContext mockDriverContext = new DriverSelectorQueryContext(userQuery,
@@ -127,6 +129,9 @@ public class TestLensDAO {
FinishedLensQuery actual = service.lensServerDao.getQuery(finishedHandle);
+ Assert.assertEquals(finishedLensQuery, actual);
+ Assert.assertEquals(actual.getConf().getProperty("prop"), "value");
+
// Try to read back result set metadata class, should not throw deserialize exception
JDBCResultSet.JDBCResultSetMetadata actualRsMeta = MAPPER.readValue(actual.getMetadata(),
JDBCResultSet.JDBCResultSetMetadata.class);