You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/10/02 09:28:10 UTC
git commit: TAJO-1081: Non-forwarded (simple) query shows wrong rows.
Repository: tajo
Updated Branches:
refs/heads/master d1022fbb8 -> 3d029fa90
TAJO-1081: Non-forwarded (simple) query shows wrong rows.
Closes #175
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/3d029fa9
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/3d029fa9
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/3d029fa9
Branch: refs/heads/master
Commit: 3d029fa90e483e6d0678b8b6c72c4a2410e97399
Parents: d1022fb
Author: Hyunsik Choi <hy...@apache.org>
Authored: Thu Oct 2 00:27:24 2014 -0700
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Thu Oct 2 00:27:24 2014 -0700
----------------------------------------------------------------------
CHANGES | 2 ++
.../tajo/cli/DefaultTajoCliOutputFormatter.java | 33 ++++++++++++++------
.../org/apache/tajo/cli/DescTableCommand.java | 6 +++-
.../java/org/apache/tajo/client/TajoClient.java | 4 ++-
.../apache/tajo/engine/planner/PlannerUtil.java | 3 +-
.../org/apache/tajo/master/GlobalEngine.java | 6 ++++
.../tajo/storage/AbstractStorageManager.java | 2 +-
7 files changed, 43 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index c73490c..3876a1e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -155,6 +155,8 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-1081: Non-forwarded (simple) query shows wrong rows. (hyunsik)
+
TAJO-981: Help command (\?) in tsql takes too long time. (YeonSu Han via jaehwa)
TAJO-962: Column reference used in LIMIT clause incurs NPE.
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
index 543ac8c..b36f521 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/DefaultTajoCliOutputFormatter.java
@@ -24,6 +24,7 @@ import org.apache.tajo.SessionVars;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.statistics.TableStats;
import org.apache.tajo.client.QueryStatus;
+import org.apache.tajo.client.TajoClient;
import org.apache.tajo.util.FileUtil;
import java.io.InputStream;
@@ -50,13 +51,25 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
this.printPause = false;
}
- private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows, String postfix) {
+ private String getQuerySuccessMessage(TableDesc tableDesc, float responseTime, int totalPrintedRows, String postfix,
+ boolean endOfTuple) {
TableStats stat = tableDesc.getStats();
- String volume = stat == null ? "0 B" : FileUtil.humanReadableByteCount(stat.getNumBytes(), false);
- long resultRows = stat == null ? 0 : stat.getNumRows();
+ String volume = stat == null ? "unknown bytes" : FileUtil.humanReadableByteCount(stat.getNumBytes(), false);
+ long resultRows = stat == null ? TajoClient.UNKNOWN_ROW_NUMBER : stat.getNumRows();
- long realNumRows = resultRows != 0 ? resultRows : totalPrintedRows;
- return "(" + realNumRows + " rows, " + getResponseTimeReadable(responseTime) + ", " + volume + " " + postfix + ")";
+ String displayRowNum;
+ if (resultRows == TajoClient.UNKNOWN_ROW_NUMBER) {
+
+ if (endOfTuple) {
+ displayRowNum = totalPrintedRows + " rows";
+ } else {
+ displayRowNum = "unknown row number";
+ }
+
+ } else {
+ displayRowNum = resultRows + " rows";
+ }
+ return "(" + displayRowNum + ", " + getResponseTimeReadable(responseTime) + ", " + volume + " " + postfix + ")";
}
protected String getResponseTimeReadable(float responseTime) {
@@ -66,13 +79,13 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
@Override
public void printResult(PrintWriter sout, InputStream sin, TableDesc tableDesc,
float responseTime, ResultSet res) throws Exception {
- long resultRows = tableDesc.getStats() == null ? 0 : tableDesc.getStats().getNumRows();
- if (resultRows == 0) {
+ long resultRows = tableDesc.getStats() == null ? -1 : tableDesc.getStats().getNumRows();
+ if (resultRows == -1) {
resultRows = Integer.MAX_VALUE;
}
if (res == null) {
- sout.println(getQuerySuccessMessage(tableDesc, responseTime, 0, "inserted"));
+ sout.println(getQuerySuccessMessage(tableDesc, responseTime, 0, "inserted", true));
return;
}
ResultSetMetaData rsmd = res.getMetaData();
@@ -86,6 +99,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
int numOfPrintedRows = 0;
int totalPrintedRows = 0;
+ boolean endOfTuple = true;
while (res.next()) {
for (int i = 1; i <= numOfColumns; i++) {
if (i > 1) sout.print(", ");
@@ -109,6 +123,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
sout.flush();
if (sin != null) {
if (sin.read() == 'q') {
+ endOfTuple = false;
sout.println();
break;
}
@@ -117,7 +132,7 @@ public class DefaultTajoCliOutputFormatter implements TajoCliOutputFormatter {
sout.println();
}
}
- sout.println(getQuerySuccessMessage(tableDesc, responseTime, totalPrintedRows, "selected"));
+ sout.println(getQuerySuccessMessage(tableDesc, responseTime, totalPrintedRows, "selected", endOfTuple));
sout.flush();
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java b/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
index d8023f2..8fab138 100644
--- a/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
+++ b/tajo-client/src/main/java/org/apache/tajo/cli/DescTableCommand.java
@@ -23,6 +23,7 @@ import org.apache.commons.lang.StringEscapeUtils;
import org.apache.tajo.catalog.Column;
import org.apache.tajo.catalog.TableDesc;
import org.apache.tajo.catalog.partition.PartitionMethodDesc;
+import org.apache.tajo.client.TajoClient;
import org.apache.tajo.util.FileUtil;
import org.apache.tajo.util.TUtil;
@@ -79,7 +80,10 @@ public class DescTableCommand extends TajoShellCommand {
sb.append("table path: ").append(desc.getPath()).append("\n");
sb.append("store type: ").append(desc.getMeta().getStoreType()).append("\n");
if (desc.getStats() != null) {
- sb.append("number of rows: ").append(desc.getStats().getNumRows()).append("\n");
+
+ long row = desc.getStats().getNumRows();
+ String rowText = row == TajoClient.UNKNOWN_ROW_NUMBER ? "unknown" : row + "";
+ sb.append("number of rows: ").append(rowText).append("\n");
sb.append("volume: ").append(
FileUtil.humanReadableByteCount(desc.getStats().getNumBytes(),
true)).append("\n");
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
----------------------------------------------------------------------
diff --git a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
index 3d61cce..2288851 100644
--- a/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
+++ b/tajo-client/src/main/java/org/apache/tajo/client/TajoClient.java
@@ -73,6 +73,8 @@ import static org.apache.tajo.ipc.ClientProtos.SerializedResultSet;
@ThreadSafe
public class TajoClient implements Closeable {
+ public static final int UNKNOWN_ROW_NUMBER = -1;
+
private final Log LOG = LogFactory.getLog(TajoClient.class);
private final TajoConf conf;
@@ -413,7 +415,7 @@ public class TajoClient implements Closeable {
}
} else {
// If a non-forwarded insert into query
- if (queryId.equals(QueryIdFactory.NULL_QUERY_ID) && response.getMaxRowNum() < 0) {
+ if (queryId.equals(QueryIdFactory.NULL_QUERY_ID) && response.getMaxRowNum() == 0) {
return this.createNullResultSet(queryId);
} else {
if (response.hasResultSet() || response.hasTableDesc()) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
index c4cc254..827be83 100644
--- a/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
+++ b/tajo-core/src/main/java/org/apache/tajo/engine/planner/PlannerUtil.java
@@ -38,6 +38,7 @@ import org.apache.tajo.engine.eval.*;
import org.apache.tajo.engine.exception.InvalidQueryException;
import org.apache.tajo.engine.planner.logical.*;
import org.apache.tajo.engine.utils.SchemaUtil;
+import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.storage.TupleComparator;
import org.apache.tajo.storage.fragment.FileFragment;
import org.apache.tajo.storage.fragment.FragmentConvertor;
@@ -854,7 +855,7 @@ public class PlannerUtil {
int startFileIndex, int numResultFiles,
AtomicInteger currentFileIndex) throws IOException {
if (fs.isDirectory(path)) {
- FileStatus[] files = fs.listStatus(path);
+ FileStatus[] files = fs.listStatus(path, StorageManager.hiddenFileFilter);
if (files != null && files.length > 0) {
for (FileStatus eachFile : files) {
if (result.size() >= numResultFiles) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
index 9612eeb..1537c89 100644
--- a/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/GlobalEngine.java
@@ -39,6 +39,7 @@ import org.apache.tajo.catalog.partition.PartitionMethodDesc;
import org.apache.tajo.catalog.proto.CatalogProtos;
import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto;
import org.apache.tajo.catalog.statistics.TableStats;
+import org.apache.tajo.client.TajoClient;
import org.apache.tajo.common.TajoDataTypes;
import org.apache.tajo.conf.TajoConf;
import org.apache.tajo.datum.DatumFactory;
@@ -750,6 +751,11 @@ public class GlobalEngine extends AbstractService {
TableStats stats = new TableStats();
stats.setNumBytes(totalSize);
+
+ if (isExternal) { // if it is an external table, there is no way to know the exact row number without processing.
+ stats.setNumRows(TajoClient.UNKNOWN_ROW_NUMBER);
+ }
+
TableDesc desc = new TableDesc(CatalogUtil.buildFQName(databaseName, simpleTableName),
schema, meta, path, isExternal);
desc.setStats(stats);
http://git-wip-us.apache.org/repos/asf/tajo/blob/3d029fa9/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
index c6be73b..4bf0057 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/AbstractStorageManager.java
@@ -304,7 +304,7 @@ public abstract class AbstractStorageManager {
// FileInputFormat Area
/////////////////////////////////////////////////////////////////////////////
- private static final PathFilter hiddenFileFilter = new PathFilter() {
+ public static final PathFilter hiddenFileFilter = new PathFilter() {
public boolean accept(Path p) {
String name = p.getName();
return !name.startsWith("_") && !name.startsWith(".");