You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by ji...@apache.org on 2016/01/28 07:26:03 UTC
tajo git commit: TAJO-2036: Prevent out of memory in the master
server, if the query result is large.
Repository: tajo
Updated Branches:
refs/heads/master 73a43d8b7 -> e9e7a36c7
TAJO-2036: Prevent out of memory in the master server, if the query result is large.
Closes #928
Signed-off-by: Jihoon Son <ji...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/e9e7a36c
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/e9e7a36c
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/e9e7a36c
Branch: refs/heads/master
Commit: e9e7a36c7b1814755e22a83e81c8ab0604064517
Parents: 73a43d8
Author: Byunghwa Yun <co...@combineads.co.kr>
Authored: Wed Jan 27 22:25:11 2016 -0800
Committer: Jihoon Son <ji...@apache.org>
Committed: Wed Jan 27 22:25:44 2016 -0800
----------------------------------------------------------------------
.../tajo/webapp/QueryExecutorServlet.java | 25 +++++----
.../resources/webapps/admin/query_executor.jsp | 54 +++++++++++---------
2 files changed, 45 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/e9e7a36c/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
index 896c83e..708c552 100644
--- a/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
+++ b/tajo-core/src/main/java/org/apache/tajo/webapp/QueryExecutorServlet.java
@@ -17,6 +17,7 @@ import org.apache.tajo.exception.TajoException;
import org.apache.tajo.ipc.ClientProtos;
import org.apache.tajo.jdbc.FetchResultSet;
import org.apache.tajo.service.ServiceTrackerFactory;
+import org.apache.tajo.util.Bytes;
import org.apache.tajo.util.JSPUtil;
import org.apache.tajo.util.TajoIdUtils;
import org.codehaus.jackson.map.DeserializationConfig;
@@ -161,6 +162,11 @@ public class QueryExecutorServlet extends HttpServlet {
} catch (java.lang.NumberFormatException nfe) {
queryRunner.sizeLimit = 1048576;
}
+ try {
+ queryRunner.rowLimit = Integer.parseInt(request.getParameter("limitRow"));
+ } catch (java.lang.NumberFormatException nfe) {
+ queryRunner.rowLimit = 3000000;
+ }
synchronized(queryRunners) {
queryRunners.put(queryRunnerId, queryRunner);
}
@@ -197,7 +203,6 @@ public class QueryExecutorServlet extends HttpServlet {
errorResponse(response, queryRunner.error);
return;
}
- returnValue.put("numOfRows", queryRunner.numOfRows);
returnValue.put("resultSize", queryRunner.resultRows);
returnValue.put("resultData", queryRunner.queryResult);
returnValue.put("resultColumns", queryRunner.columnNames);
@@ -286,7 +291,7 @@ public class QueryExecutorServlet extends HttpServlet {
String database;
long resultRows;
int sizeLimit;
- long numOfRows;
+ long rowLimit;
Exception error;
AtomicInteger progress = new AtomicInteger(0);
@@ -501,20 +506,20 @@ public class QueryExecutorServlet extends HttpServlet {
}
queryResult = new ArrayList<>();
- if(sizeLimit < resultRows) {
- numOfRows = (long)((float)(resultRows) * ((float)sizeLimit / (float) resultRows));
- } else {
- numOfRows = resultRows;
- }
-
+ int currentResultSize = 0;
int rowCount = 0;
while (res.next()) {
- if(rowCount > numOfRows) {
+ if(rowCount > rowLimit || currentResultSize > sizeLimit) {
break;
}
List<Object> row = new ArrayList<>();
for(int i = 0; i < numOfColumns; i++) {
- row.add(String.valueOf(res.getObject(i + 1)));
+ String columnValue = String.valueOf(res.getObject(i + 1));
+ try {
+ currentResultSize += columnValue.getBytes(Bytes.UTF8_ENCODING).length;
+ } catch (Exception e) {
+ }
+ row.add(columnValue);
}
queryResult.add(row);
rowCount++;
http://git-wip-us.apache.org/repos/asf/tajo/blob/e9e7a36c/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
index 6b6a3db..f359c99 100644
--- a/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
+++ b/tajo-core/src/main/resources/webapps/admin/query_executor.jsp
@@ -64,6 +64,7 @@ var progressTimer = null;
var queryRunnerId = null;
var PRINT_LIMIT = 25;
var SIZE_LIMIT = 104857600; // Limit size of displayed results.(Bytes)
+var ROW_LIMIT = 3000000;
var pageNum = 0;
var pageCount, storedColumns, storedData;
@@ -87,6 +88,9 @@ function runQuery() {
} else if(Math.ceil(Number($("#sizeLimit").val())) > 0) {
SIZE_LIMIT = Number($("#sizeLimit").val()) * 1024 * 1024;
}
+ if(Math.ceil(Number($("#rowLimit").val())) > 0) {
+ ROW_LIMIT = Number($("#rowLimit").val()) * 1000 * 1000;
+ }
if(Math.ceil(Number($("#printLimit").val())) > 0) {
PRINT_LIMIT = Number($("#printLimit").val());
}
@@ -101,7 +105,7 @@ function runQuery() {
$.ajax({
type: "POST",
url: "query_exec",
- data: { action: "runQuery", query: query, prevQueryId: queryRunnerId, limitSize:SIZE_LIMIT, database: sbox.options[sbox.selectedIndex].text }
+ data: { action: "runQuery", query: query, prevQueryId: queryRunnerId, limitSize:SIZE_LIMIT, limitRow:ROW_LIMIT, database: sbox.options[sbox.selectedIndex].text }
})
.done(function(msg) {
var resultJson = $.parseJSON(msg);
@@ -234,26 +238,26 @@ function getCSV() {
}
function getNext() {
- var printedLine = 0;
- if(pageCount > pageNum) {
- pageNum++;
- document.getElementById("selectPage").options.selectedIndex = pageNum;
- }else {
- alert("There's no next page.");
- return;
- }
- getPage();
+ var printedLine = 0;
+ if(pageCount > pageNum) {
+ pageNum++;
+ document.getElementById("selectPage").options.selectedIndex = pageNum;
+ } else {
+ alert("There's no next page.");
+ return;
+ }
+ getPage();
}
function getPrev() {
- if(pageNum > 0 ) {
- pageNum--;
- document.getElementById("selectPage").options.selectedIndex = pageNum;
- } else {
- alert("There's no previous page.");
- return;
- }
- getPage();
+ if(pageNum > 0 ) {
+ pageNum--;
+ document.getElementById("selectPage").options.selectedIndex = pageNum;
+ } else {
+ alert("There's no previous page.");
+ return;
+ }
+ getPage();
}
function getSelectedPage() {
@@ -298,18 +302,20 @@ function getPage() {
Database :
<select id="selectDatabase" name="database" width="190" style="width: 190px">
<%
- for (String databaseName : master.getCatalog().getAllDatabaseNames()) {
- %>
- <option value="<%=databaseName%>" <%= (databaseName.equals("default"))?"selected":"" %> ><%=databaseName%></option>
- <%
- }
- %>
+ for (String databaseName : master.getCatalog().getAllDatabaseNames()) {
+ %>
+ <option value="<%=databaseName%>" <%= (databaseName.equals("default"))?"selected":"" %> ><%=databaseName%></option>
+ <%
+ }
+ %>
</select>
<p />
<textarea id="query" style="width:800px; height:250px; font-family:Tahoma; font-size:12px;"></textarea>
<p />
Limit : <input id="sizeLimit" type="text" value="10" style="width:30px; text-align:center;" /> MB
<p />
+ Limit Rows : <input id="rowLimit" type="text" value="3" style="width:30px; text-align:center;" /> M
+ <p />
Rows/Page : <input id="printLimit" type="text" value="25" style="width:30px; text-align:center;" />
<hr />
<input id="btnSubmit" type="submit" value="Submit">