You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tajo.apache.org by hy...@apache.org on 2014/07/09 06:10:50 UTC
[33/50] [abbrv] git commit: TAJO-874: Sometimes
InvalidOperationException occurs when aggregates TableStat. (Hyoungjun Kim
via hyunsik)
TAJO-874: Sometimes InvalidOperationException occurs when aggregates TableStat. (Hyoungjun Kim via hyunsik)
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/4165f339
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/4165f339
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/4165f339
Branch: refs/heads/window_function
Commit: 4165f33984368c3f0ff504ff65a3f3c0d6904712
Parents: 26265c0
Author: Hyunsik Choi <hy...@apache.org>
Authored: Wed Jun 25 17:58:29 2014 +0900
Committer: Hyunsik Choi <hy...@apache.org>
Committed: Wed Jun 25 17:58:29 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../tajo/catalog/statistics/StatisticsUtil.java | 44 ++++++++++++--------
.../java/org/apache/tajo/datum/CharDatum.java | 4 +-
.../java/org/apache/tajo/datum/DateDatum.java | 2 +-
.../java/org/apache/tajo/datum/Float4Datum.java | 6 +--
.../java/org/apache/tajo/datum/Float8Datum.java | 6 +--
.../java/org/apache/tajo/datum/Int2Datum.java | 4 +-
.../java/org/apache/tajo/datum/Int4Datum.java | 6 +--
.../java/org/apache/tajo/datum/Int8Datum.java | 4 +-
.../org/apache/tajo/datum/IntervalDatum.java | 3 +-
.../java/org/apache/tajo/datum/TextDatum.java | 2 +-
.../java/org/apache/tajo/datum/TimeDatum.java | 4 +-
.../org/apache/tajo/datum/TimestampDatum.java | 4 +-
.../apache/tajo/storage/TableStatistics.java | 17 +++++++-
14 files changed, 66 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index c10dbdf..a80986c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -74,6 +74,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-874: Sometimes InvalidOperationException occurs when aggregates
+ TableStat. (Hyoungjun Kim via hyunsik)
+
TAJO-866: COUNT DISTINCT with other aggregation function throws
ClassCastException. (Hyoungjun Kim via hyunsik)
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
index 01316bc..c481276 100644
--- a/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
+++ b/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/statistics/StatisticsUtil.java
@@ -65,15 +65,19 @@ public class StatisticsUtil {
continue;
}
- agg.setNumDistVals(agg.getNumDistValues() + cs.getNumDistValues());
- agg.setNumNulls(agg.getNumNulls() + cs.getNumNulls());
- if (!cs.minIsNotSet() && (agg.minIsNotSet() ||
- agg.getMinValue().compareTo(cs.getMinValue()) > 0)) {
- agg.setMinValue(cs.getMinValue());
- }
- if (!cs.maxIsNotSet() && (agg.maxIsNotSet() ||
- agg.getMaxValue().compareTo(cs.getMaxValue()) < 0)) {
- agg.setMaxValue(stats.getColumnStats().get(i).getMaxValue());
+ try {
+ agg.setNumDistVals(agg.getNumDistValues() + cs.getNumDistValues());
+ agg.setNumNulls(agg.getNumNulls() + cs.getNumNulls());
+ if (!cs.minIsNotSet() && (agg.minIsNotSet() ||
+ agg.getMinValue().compareTo(cs.getMinValue()) > 0)) {
+ agg.setMinValue(cs.getMinValue());
+ }
+ if (!cs.maxIsNotSet() && (agg.maxIsNotSet() ||
+ agg.getMaxValue().compareTo(cs.getMaxValue()) < 0)) {
+ agg.setMaxValue(stats.getColumnStats().get(i).getMaxValue());
+ }
+ } catch (Exception e) {
+ LOG.warn(e.getMessage(), e);
}
}
}
@@ -117,15 +121,19 @@ public class StatisticsUtil {
LOG.warn("ERROR: One of column stats is NULL (expected column: " + css[i].getColumn() + ")");
continue;
}
- css[i].setNumDistVals(css[i].getNumDistValues() + cs.getNumDistValues());
- css[i].setNumNulls(css[i].getNumNulls() + cs.getNumNulls());
- if (!cs.minIsNotSet() && (css[i].minIsNotSet() ||
- css[i].getMinValue().compareTo(cs.getMinValue()) > 0)) {
- css[i].setMinValue(cs.getMinValue());
- }
- if (!cs.maxIsNotSet() && (css[i].maxIsNotSet() ||
- css[i].getMaxValue().compareTo(cs.getMaxValue()) < 0)) {
- css[i].setMaxValue(ts.getColumnStats().get(i).getMaxValue());
+ try {
+ css[i].setNumDistVals(css[i].getNumDistValues() + cs.getNumDistValues());
+ css[i].setNumNulls(css[i].getNumNulls() + cs.getNumNulls());
+ if (!cs.minIsNotSet() && (css[i].minIsNotSet() ||
+ css[i].getMinValue().compareTo(cs.getMinValue()) > 0)) {
+ css[i].setMinValue(cs.getMinValue());
+ }
+ if (!cs.maxIsNotSet() && (css[i].maxIsNotSet() ||
+ css[i].getMaxValue().compareTo(cs.getMaxValue()) < 0)) {
+ css[i].setMaxValue(ts.getColumnStats().get(i).getMaxValue());
+ }
+ } catch (Exception e) {
+ LOG.warn(e.getMessage(), e);
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
index 444ca66..e6c4d94 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/CharDatum.java
@@ -141,7 +141,7 @@ public class CharDatum extends Datum {
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -156,7 +156,7 @@ public class CharDatum extends Datum {
return -1;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
index 10f7077..b26ef84 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/DateDatum.java
@@ -254,7 +254,7 @@ public class DateDatum extends Datum {
} else if (datum instanceof NullDatum || datum.isNull()) {
return -1;
} else {
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
index c269a40..e24bce4 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Float4Datum.java
@@ -135,7 +135,7 @@ public class Float4Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -195,7 +195,7 @@ public class Float4Datum extends NumericDatum {
case NULL_TYPE:
return -1;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -268,7 +268,7 @@ public class Float4Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
index b4cfc67..0542148 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Float8Datum.java
@@ -124,7 +124,7 @@ public class Float8Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -184,7 +184,7 @@ public class Float8Datum extends NumericDatum {
case NULL_TYPE:
return -1;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -257,7 +257,7 @@ public class Float8Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
index 7770b94..38cf019 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int2Datum.java
@@ -126,7 +126,7 @@ public class Int2Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -186,7 +186,7 @@ public class Int2Datum extends NumericDatum {
case NULL_TYPE:
return -1;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
index 7f7ac13..d26b6b2 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int4Datum.java
@@ -130,7 +130,7 @@ public class Int4Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -190,7 +190,7 @@ public class Int4Datum extends NumericDatum {
case NULL_TYPE:
return -1;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -263,7 +263,7 @@ public class Int4Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java b/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
index 3b1e67d..46a1353 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/Int8Datum.java
@@ -137,7 +137,7 @@ public class Int8Datum extends NumericDatum {
case NULL_TYPE:
return datum;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -198,7 +198,7 @@ public class Int8Datum extends NumericDatum {
return -1;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
index 5ad9635..28a0a47 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/IntervalDatum.java
@@ -96,7 +96,6 @@ public class IntervalDatum extends Datum {
int length = intervalStr.getBytes().length;
- int start = 0;
StringBuilder digitChars = new StringBuilder();
StringBuilder unitChars = new StringBuilder();
for (int i = 0; i < length; i++) {
@@ -406,7 +405,7 @@ public class IntervalDatum extends Datum {
} else if (datum instanceof NullDatum || datum.isNull()) {
return -1;
} else {
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
index 313b905..49f09f6 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TextDatum.java
@@ -110,7 +110,7 @@ public class TextDatum extends Datum {
return -1;
default:
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java
index a7b9d1e..37e5e78 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimeDatum.java
@@ -167,7 +167,7 @@ public class TimeDatum extends Datum {
} else if (datum.isNull()) {
return datum;
} else {
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -179,7 +179,7 @@ public class TimeDatum extends Datum {
} else if (datum instanceof NullDatum || datum.isNull()) {
return -1;
} else {
- throw new InvalidOperationException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
----------------------------------------------------------------------
diff --git a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
index d3cc5f4..62b0861 100644
--- a/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
+++ b/tajo-common/src/main/java/org/apache/tajo/datum/TimestampDatum.java
@@ -170,7 +170,7 @@ public class TimestampDatum extends Datum {
} else if (datum.isNull()) {
return datum;
} else {
- throw new RuntimeException();
+ throw new InvalidOperationException(datum.type());
}
}
@@ -182,7 +182,7 @@ public class TimestampDatum extends Datum {
} else if (datum.isNull()) {
return -1;
} else {
- throw new RuntimeException();
+ throw new InvalidOperationException(datum.type());
}
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4165f339/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
----------------------------------------------------------------------
diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java b/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
index cbee95e..ac9bd8a 100644
--- a/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
+++ b/tajo-storage/src/main/java/org/apache/tajo/storage/TableStatistics.java
@@ -18,6 +18,8 @@
package org.apache.tajo.storage;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.statistics.ColumnStats;
import org.apache.tajo.catalog.statistics.TableStats;
@@ -30,6 +32,7 @@ import org.apache.tajo.datum.NullDatum;
* This class is not thread-safe.
*/
public class TableStatistics {
+ private static final Log LOG = LogFactory.getLog(TableStatistics.class);
private Schema schema;
private Tuple minValues;
private Tuple maxValues;
@@ -104,8 +107,18 @@ public class TableStatistics {
for (int i = 0; i < schema.size(); i++) {
columnStats = new ColumnStats(schema.getColumn(i));
columnStats.setNumNulls(numNulls[i]);
- columnStats.setMinValue(minValues.get(i));
- columnStats.setMaxValue(maxValues.get(i));
+ if (minValues.get(i) == null || schema.getColumn(i).getDataType().getType() == minValues.get(i).type()) {
+ columnStats.setMinValue(minValues.get(i));
+ } else {
+ LOG.warn("Wrong statistics column type (" + minValues.get(i).type() +
+ ", expected=" + schema.getColumn(i).getDataType().getType() + ")");
+ }
+ if (minValues.get(i) == null || schema.getColumn(i).getDataType().getType() == minValues.get(i).type()) {
+ columnStats.setMaxValue(maxValues.get(i));
+ } else {
+ LOG.warn("Wrong statistics column type (" + minValues.get(i).type() +
+ ", expected=" + schema.getColumn(i).getDataType().getType() + ")");
+ }
stat.addColumnStat(columnStats);
}