You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lens.apache.org by pr...@apache.org on 2015/08/10 15:35:33 UTC
incubator-lens git commit: LENS-651: Include File size in Query Result
Repository: incubator-lens
Updated Branches:
refs/heads/master cbdbea8f2 -> 4d3e9e67d
LENS-651: Include File size in Query Result
Project: http://git-wip-us.apache.org/repos/asf/incubator-lens/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-lens/commit/4d3e9e67
Tree: http://git-wip-us.apache.org/repos/asf/incubator-lens/tree/4d3e9e67
Diff: http://git-wip-us.apache.org/repos/asf/incubator-lens/diff/4d3e9e67
Branch: refs/heads/master
Commit: 4d3e9e67d64bb3b6fb4f3e72a52bfe39304ca554
Parents: cbdbea8
Author: Deepak Barr <de...@gmail.com>
Authored: Mon Aug 10 19:04:51 2015 +0530
Committer: Rajat Khandelwal <pr...@apache.org>
Committed: Mon Aug 10 19:04:51 2015 +0530
----------------------------------------------------------------------
.../lens/api/query/PersistentQueryResult.java | 10 ++++++-
.../lens/cli/commands/LensQueryCommands.java | 9 ++++--
.../lens/driver/es/client/ESResultSet.java | 4 +--
.../lens/driver/hive/HiveInMemoryResultSet.java | 4 +--
.../driver/hive/HivePersistentResultSet.java | 9 ++++--
.../apache/lens/driver/jdbc/JDBCResultSet.java | 4 +--
.../lens/lib/query/AbstractFileFormatter.java | 14 +++++++--
.../lens/lib/query/HadoopFileFormatter.java | 2 ++
.../lens/lib/query/WrappedFileFormatter.java | 7 ++++-
.../apache/lens/lib/query/ZipFileFormatter.java | 2 ++
.../lens/server/api/driver/LensResultSet.java | 2 +-
.../server/api/driver/PersistentResultSet.java | 11 ++++++-
.../server/api/query/FinishedLensQuery.java | 9 +++++-
.../server/api/query/QueryOutputFormatter.java | 10 +++++--
.../lens/server/api/driver/MockDriver.java | 14 ++++++---
.../lens/server/query/LensPersistentResult.java | 15 ++++++++--
.../apache/lens/server/query/LensServerDAO.java | 13 +++++----
.../server/query/QueryExecutionServiceImpl.java | 9 ++++--
.../lens/server/query/TestQueryService.java | 30 ++++++++++++++++++++
19 files changed, 142 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-api/src/main/java/org/apache/lens/api/query/PersistentQueryResult.java
----------------------------------------------------------------------
diff --git a/lens-api/src/main/java/org/apache/lens/api/query/PersistentQueryResult.java b/lens-api/src/main/java/org/apache/lens/api/query/PersistentQueryResult.java
index 70dac08..a2e10c6 100644
--- a/lens-api/src/main/java/org/apache/lens/api/query/PersistentQueryResult.java
+++ b/lens-api/src/main/java/org/apache/lens/api/query/PersistentQueryResult.java
@@ -60,5 +60,13 @@ public class PersistentQueryResult extends QueryResult {
*/
@XmlElement
@Getter
- private int numRows;
+ private Integer numRows;
+
+ /**
+ * The file size.
+ */
+ @XmlElement
+ @Getter
+ private Long fileSize;
+
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
----------------------------------------------------------------------
diff --git a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
index 88facda..7a5b177 100644
--- a/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
+++ b/lens-cli/src/main/java/org/apache/lens/cli/commands/LensQueryCommands.java
@@ -127,17 +127,20 @@ public class LensQueryCommands extends BaseLensCommand {
numRows++;
b.append("\n");
}
+ b.append(numRows + " rows ");
} else {
PersistentQueryResult temp = (PersistentQueryResult) r;
- numRows = temp.getNumRows();
- b.append("Results of query stored at : ").append(temp.getPersistedURI()).append(" ");
+ b.append("Results of query stored at : ").append(temp.getPersistedURI()).append(" ");
+ if (null != temp.getNumRows()) {
+ b.append(temp.getNumRows() + " rows ");
+ }
}
}
if (rs.getQuery() != null) {
long submissionTime = rs.getQuery().getSubmissionTime();
long endTime = rs.getQuery().getFinishTime();
- b.append(numRows).append(" rows process in (").append(endTime > 0 ? ((endTime - submissionTime) / 1000) : 0)
+ b.append("processed in (").append(endTime > 0 ? ((endTime - submissionTime) / 1000) : 0)
.append(") seconds.\n");
}
return b.toString();
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/ESResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/ESResultSet.java b/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/ESResultSet.java
index 4ba2321..464b535 100644
--- a/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/ESResultSet.java
+++ b/lens-driver-es/src/main/java/org/apache/lens/driver/es/client/ESResultSet.java
@@ -36,7 +36,7 @@ public class ESResultSet extends InMemoryResultSet {
final Iterator<ResultRow> resultSetIterator;
@NonNull
final LensResultSetMetadata resultSetMetadata;
- final int size;
+ final Integer size;
public ESResultSet(int size, final Iterable<ResultRow> resultSetIterable, final LensResultSetMetadata metadata) {
this.size = size;
@@ -60,7 +60,7 @@ public class ESResultSet extends InMemoryResultSet {
}
@Override
- public int size() {
+ public Integer size() {
return size;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
index 3d1f20a..f8abd78 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HiveInMemoryResultSet.java
@@ -90,8 +90,8 @@ public class HiveInMemoryResultSet extends InMemoryResultSet {
* @see org.apache.lens.server.api.driver.LensResultSet#size()
*/
@Override
- public int size() throws LensException {
- return -1;
+ public Integer size() throws LensException {
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
index 82b0568..00e1e53 100644
--- a/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
+++ b/lens-driver-hive/src/main/java/org/apache/lens/driver/hive/HivePersistentResultSet.java
@@ -59,8 +59,13 @@ public class HivePersistentResultSet extends PersistentResultSet {
* @see org.apache.lens.server.api.driver.LensResultSet#size()
*/
@Override
- public int size() throws LensException {
- return -1;
+ public Integer size() throws LensException {
+ return null;
+ }
+
+ @Override
+ public Long fileSize() throws LensException {
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
----------------------------------------------------------------------
diff --git a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
index f80cdd8..8b4da3f 100644
--- a/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
+++ b/lens-driver-jdbc/src/main/java/org/apache/lens/driver/jdbc/JDBCResultSet.java
@@ -99,9 +99,9 @@ public class JDBCResultSet extends InMemoryResultSet {
* @see org.apache.lens.server.api.driver.LensResultSet#size()
*/
@Override
- public int size() throws LensException {
+ public Integer size() throws LensException {
log.warn("Size of result set is not supported");
- return -1;
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
index 48e020f..ae5af03 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/AbstractFileFormatter.java
@@ -33,7 +33,12 @@ public abstract class AbstractFileFormatter extends AbstractOutputFormatter impl
/**
* The num rows.
*/
- protected int numRows = 0;
+ protected Integer numRows;
+
+ /**
+ * The file size.
+ */
+ protected Long fileSize;
/**
* The final path.
@@ -53,10 +58,15 @@ public abstract class AbstractFileFormatter extends AbstractOutputFormatter impl
}
@Override
- public int getNumRows() {
+ public Integer getNumRows() {
return numRows;
}
+ @Override
+ public Long getFileSize() {
+ return fileSize;
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
index 2c48209..6f55c79 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/HadoopFileFormatter.java
@@ -60,6 +60,7 @@ public class HadoopFileFormatter extends AbstractFileFormatter {
try {
rowWriter = LensFileOutputFormat.createRecordWriter(ctx.getConf(), tmpWorkPath, Reporter.NULL,
ctx.getCompressOutput(), ctx.getOuptutFileExtn(), ctx.getResultEncoding());
+ numRows=0;
} catch (IOException e) {
throw new IllegalArgumentException("Could not create tmp path");
}
@@ -119,6 +120,7 @@ public class HadoopFileFormatter extends AbstractFileFormatter {
finalPath = finalPath.makeQualified(fs);
fs.rename(rowWriter.getTmpPath(), finalPath);
ctx.setResultSetPath(finalPath.toString());
+ fileSize = fs.getFileStatus(finalPath).getLen();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
index aa66b7b..e28c17b 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/WrappedFileFormatter.java
@@ -94,10 +94,15 @@ public abstract class WrappedFileFormatter extends AbstractOutputFormatter {
}
@Override
- public int getNumRows() {
+ public Integer getNumRows() {
return formatter.getNumRows();
}
+ @Override
+ public Long getFileSize() {
+ return formatter.getFileSize();
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
----------------------------------------------------------------------
diff --git a/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java b/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
index cc6d918..a3614fe 100644
--- a/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
+++ b/lens-query-lib/src/main/java/org/apache/lens/lib/query/ZipFileFormatter.java
@@ -93,6 +93,7 @@ public class ZipFileFormatter extends AbstractFileFormatter {
public void setupOutputs() throws IOException {
resultFileExtn = ctx.getOuptutFileExtn();
maxSplitRows = ctx.getMaxResultSplitRows();
+ numRows = 0;
String pathStr = ctx.getResultSetParentDir();
if (StringUtils.isBlank(pathStr)) {
@@ -131,6 +132,7 @@ public class ZipFileFormatter extends AbstractFileFormatter {
close();
fs.rename(tmpPath, finalPath);
finalPath = finalPath.makeQualified(fs);
+ fileSize = fs.getFileStatus(finalPath).getLen();
ctx.setResultSetPath(getFinalOutputPath());
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensResultSet.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensResultSet.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensResultSet.java
index af354cc..929a302 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensResultSet.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/LensResultSet.java
@@ -32,7 +32,7 @@ public abstract class LensResultSet {
* @return The size if available, -1 if not available.
* @throws LensException the lens exception
*/
- public abstract int size() throws LensException;
+ public abstract Integer size() throws LensException;
/**
* Get the result set metadata
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server-api/src/main/java/org/apache/lens/server/api/driver/PersistentResultSet.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/PersistentResultSet.java b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/PersistentResultSet.java
index 13f069f..bbde170 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/driver/PersistentResultSet.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/driver/PersistentResultSet.java
@@ -26,6 +26,15 @@ import org.apache.lens.server.api.error.LensException;
* The Class PersistentResultSet.
*/
public abstract class PersistentResultSet extends LensResultSet {
+
+ /**
+ * Get the size of the result set file.
+ *
+ * @return The size if available, null if not available.
+ * @throws LensException the lens exception
+ */
+ public abstract Long fileSize() throws LensException;
+
public abstract String getOutputPath() throws LensException;
/*
@@ -34,6 +43,6 @@ public abstract class PersistentResultSet extends LensResultSet {
* @see org.apache.lens.server.api.driver.LensResultSet#toQueryResult()
*/
public QueryResult toQueryResult() throws LensException {
- return new PersistentQueryResult(getOutputPath(), size());
+ return new PersistentQueryResult(getOutputPath(), size(), fileSize());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
index 0c78e01..6cecf7e 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/FinishedLensQuery.java
@@ -121,7 +121,14 @@ public class FinishedLensQuery {
*/
@Getter
@Setter
- private int rows;
+ private Integer rows;
+
+ /**
+ * The file size.
+ */
+ @Getter
+ @Setter
+ private Long fileSize;
/**
* The error message.
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
index 1d2e89b..0a6cc6b 100644
--- a/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
+++ b/lens-server-api/src/main/java/org/apache/lens/server/api/query/QueryOutputFormatter.java
@@ -79,9 +79,15 @@ public interface QueryOutputFormatter {
/**
* Get total number of rows in result.
*
- * @return Total number of rows, return -1, if not known
+ * @return Total number of rows, return null, if not known
*/
- int getNumRows();
+ Integer getNumRows();
+
+ /**
+ * Get size of the resultset file.
+ * @return Total number of rows, return null, if not known
+ */
+ Long getFileSize();
/**
* Get resultset metadata
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
----------------------------------------------------------------------
diff --git a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
index 17dc99b..b28669e 100644
--- a/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
+++ b/lens-server-api/src/test/java/org/apache/lens/server/api/driver/MockDriver.java
@@ -249,9 +249,15 @@ public class MockDriver implements LensDriver {
return new PersistentResultSet() {
@Override
- public int size() throws LensException {
+ public Integer size() throws LensException {
// TODO Auto-generated method stub
- return 0;
+ return null;
+ }
+
+ @Override
+ public Long fileSize() throws LensException {
+ // TODO Auto-generated method stub
+ return null;
}
@Override
@@ -295,9 +301,9 @@ public class MockDriver implements LensDriver {
return new InMemoryResultSet() {
@Override
- public int size() throws LensException {
+ public Integer size() throws LensException {
// TODO Auto-generated method stub
- return 0;
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
index 80eb697..b65a5f4 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensPersistentResult.java
@@ -34,7 +34,10 @@ public class LensPersistentResult extends PersistentResultSet {
private final String outputPath;
/** The num rows. */
- private final int numRows;
+ private final Integer numRows;
+
+ /** The file size. */
+ private final Long fileSize;
/**
* Instantiates a new lens persistent result.
@@ -43,10 +46,11 @@ public class LensPersistentResult extends PersistentResultSet {
* @param outputPath the output path
* @param numRows the num rows
*/
- public LensPersistentResult(LensResultSetMetadata metadata, String outputPath, int numRows) {
+ public LensPersistentResult(LensResultSetMetadata metadata, String outputPath, Integer numRows, Long fileSize) {
this.metadata = metadata;
this.outputPath = outputPath;
this.numRows = numRows;
+ this.fileSize = fileSize;
}
@Override
@@ -60,11 +64,16 @@ public class LensPersistentResult extends PersistentResultSet {
* @see org.apache.lens.server.api.driver.LensResultSet#size()
*/
@Override
- public int size() throws LensException {
+ public Integer size() throws LensException {
return numRows;
}
@Override
+ public Long fileSize() throws LensException {
+ return fileSize;
+ }
+
+ @Override
public LensResultSetMetadata getMetadata() throws LensException {
return metadata;
}
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
index e392e34..59b1221 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/LensServerDAO.java
@@ -84,8 +84,9 @@ public class LensServerDAO {
String sql = "CREATE TABLE if not exists finished_queries (handle varchar(255) not null unique,"
+ "userquery varchar(10000) not null," + "submitter varchar(255) not null," + "starttime bigint, "
+ "endtime bigint," + "result varchar(255)," + "status varchar(255), " + "metadata varchar(100000), "
- + "rows int, " + "errormessage varchar(10000), " + "driverstarttime bigint, " + "driverendtime bigint, "
- + "metadataclass varchar(10000)," + "queryname varchar(255)," + "submissiontime bigint" + ")";
+ + "rows int, " + "filesize bigint, " + "errormessage varchar(10000), " + "driverstarttime bigint, "
+ + "driverendtime bigint, " + "metadataclass varchar(10000), " + "queryname varchar(255), "
+ + "submissiontime bigint" + ")";
try {
QueryRunner runner = new QueryRunner(ds);
runner.update(sql);
@@ -107,16 +108,16 @@ public class LensServerDAO {
// The expected case
Connection conn = null;
String sql = "insert into finished_queries (handle, userquery,submitter,"
- + "starttime,endtime,result,status,metadata,rows,"
+ + "starttime,endtime,result,status,metadata,rows,filesize,"
+ "errormessage,driverstarttime,driverendtime, metadataclass, queryname, submissiontime)"
- + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
+ + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)";
try {
conn = getConnection();
QueryRunner runner = new QueryRunner();
runner.update(conn, sql, query.getHandle(), query.getUserQuery(), query.getSubmitter(), query.getStartTime(),
query.getEndTime(), query.getResult(), query.getStatus(), query.getMetadata(), query.getRows(),
- query.getErrorMessage(), query.getDriverStartTime(), query.getDriverEndTime(), query.getMetadataClass(),
- query.getQueryName(), query.getSubmissionTime());
+ query.getFileSize(), query.getErrorMessage(), query.getDriverStartTime(), query.getDriverEndTime(),
+ query.getMetadataClass(), query.getQueryName(), query.getSubmissionTime());
conn.commit();
} finally {
DbUtils.closeQuietly(conn);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
----------------------------------------------------------------------
diff --git a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
index c773534..c29a1ac 100644
--- a/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
+++ b/lens-server/src/main/java/org/apache/lens/server/query/QueryExecutionServiceImpl.java
@@ -913,11 +913,13 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
if (set != null && PersistentResultSet.class.isAssignableFrom(set.getClass())) {
LensResultSetMetadata metadata = set.getMetadata();
String outputPath = ((PersistentResultSet) set).getOutputPath();
- int rows = set.size();
+ Long fileSize = ((PersistentResultSet) set).fileSize();
+ Integer rows = set.size();
finishedQuery.setMetadataClass(metadata.getClass().getName());
finishedQuery.setResult(outputPath);
finishedQuery.setMetadata(MAPPER.writeValueAsString(metadata));
finishedQuery.setRows(rows);
+ finishedQuery.setFileSize(fileSize);
}
}
}
@@ -1419,7 +1421,7 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
try {
Class<LensResultSetMetadata> mdKlass = (Class<LensResultSetMetadata>) Class.forName(query.getMetadataClass());
return new LensPersistentResult(MAPPER.readValue(query.getMetadata(), mdKlass), query.getResult(),
- query.getRows());
+ query.getRows(), query.getFileSize());
} catch (Exception e) {
throw new LensException(e);
}
@@ -1451,7 +1453,8 @@ public class QueryExecutionServiceImpl extends BaseLensService implements QueryE
new LensPersistentResult(
ctx.getQueryOutputFormatter().getMetadata(),
ctx.getQueryOutputFormatter().getFinalOutputPath(),
- ctx.getQueryOutputFormatter().getNumRows()));
+ ctx.getQueryOutputFormatter().getNumRows(),
+ ctx.getQueryOutputFormatter().getFileSize()));
} else if (allQueries.get(queryHandle).isResultAvailableInDriver()) {
resultSet = allQueries.get(queryHandle).getSelectedDriver().fetchResultSet(allQueries.get(queryHandle));
resultSets.put(queryHandle, resultSet);
http://git-wip-us.apache.org/repos/asf/incubator-lens/blob/4d3e9e67/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
----------------------------------------------------------------------
diff --git a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
index bdcb46c..131f008 100644
--- a/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
+++ b/lens-server/src/test/java/org/apache/lens/server/query/TestQueryService.java
@@ -802,6 +802,31 @@ public class TestQueryService extends LensJerseyTest {
}
/**
+ * Returns the size of result set file when result path is a file, null otherwise
+ *
+ * @param resultset
+ * @param handle
+ * @param isDir
+ * @return
+ * @throws IOException
+ */
+ public static Long readResultFileSize(PersistentQueryResult resultset, QueryHandle handle, boolean isDir)
+ throws IOException {
+ assertTrue(resultset.getPersistedURI().contains(handle.toString()));
+ Path actualPath = new Path(resultset.getPersistedURI());
+ FileSystem fs = actualPath.getFileSystem(new Configuration());
+ FileStatus fileStatus = fs.getFileStatus(actualPath);
+ if (fileStatus.isDir()) {
+ assertTrue(isDir);
+ return null;
+ } else {
+ assertFalse(isDir);
+ return fileStatus.getLen();
+ }
+ }
+
+
+ /**
* Adds the rows from file.
*
* @param actualRows the actual rows
@@ -841,6 +866,11 @@ public class TestQueryService extends LensJerseyTest {
throws IOException {
List<String> actualRows = readResultSet(resultset, handle, isDir);
validatePersistentResult(actualRows);
+ if (!isDir) {
+ assertEquals(resultset.getNumRows().intValue(), actualRows.size());
+ }
+ Long fileSize = readResultFileSize(resultset, handle, isDir);
+ assertEquals(resultset.getFileSize(), fileSize);
}
static void validatePersistentResult(List<String> actualRows) {