You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tsfile.apache.org by ji...@apache.org on 2025/01/15 10:44:47 UTC
(tsfile) branch statistic_compatibility_pr created (now 72800c23)
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a change to branch statistic_compatibility_pr
in repository https://gitbox.apache.org/repos/asf/tsfile.git
at 72800c23 add column compatibility
This branch includes the following new commits:
new 5308a4f2 add statistic compatibility
new f65a1f6e fix test
new 72800c23 add column compatibility
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tsfile) 03/03: add column compatibility
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch statistic_compatibility_pr
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 72800c23fc1f25e53ec5471b5cc5605ab5f472b3
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Jan 13 13:11:05 2025 +0800
add column compatibility
(cherry picked from commit 8dd5f1c18b0b0c0e0268ea3fe04f5c9ed953058c)
---
.../read/common/block/column/FloatColumn.java | 15 ++++++++
.../tsfile/read/common/block/column/IntColumn.java | 43 ++++++++++++++++++++++
.../read/common/block/column/LongColumn.java | 15 ++++++++
3 files changed, 73 insertions(+)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java
index 98bcd90e..c008d935 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/FloatColumn.java
@@ -34,6 +34,7 @@ import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkValidRe
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfFloatArray;
+@SuppressWarnings("java:S3012")
public class FloatColumn implements Column {
private static final int INSTANCE_SIZE =
@@ -94,11 +95,25 @@ public class FloatColumn implements Column {
return values[position + arrayOffset];
}
+ @Override
+ public double getDouble(int position) {
+ return values[position + arrayOffset];
+ }
+
@Override
public float[] getFloats() {
return values;
}
+ @Override
+ public double[] getDoubles() {
+ double[] doubles = new double[values.length];
+ for (int i = 0; i < values.length; i++) {
+ doubles[i] = values[i];
+ }
+ return doubles;
+ }
+
@Override
public Object getObject(int position) {
return getFloat(position);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java
index 3e0caa91..c2065ee3 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/IntColumn.java
@@ -34,6 +34,7 @@ import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkValidRe
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfIntArray;
+@SuppressWarnings("java:S3012")
public class IntColumn implements Column {
private static final int INSTANCE_SIZE =
@@ -94,11 +95,53 @@ public class IntColumn implements Column {
return values[position + arrayOffset];
}
+ @Override
+ public long getLong(int position) {
+ return values[position + arrayOffset];
+ }
+
+ @Override
+ public float getFloat(int position) {
+ return values[position + arrayOffset];
+ }
+
+ @Override
+ public double getDouble(int position) {
+ return values[position + arrayOffset];
+ }
+
@Override
public int[] getInts() {
return values;
}
+ @Override
+ public float[] getFloats() {
+ float[] result = new float[values.length];
+ for (int i = 0; i < values.length; i++) {
+ result[i] = values[i];
+ }
+ return result;
+ }
+
+ @Override
+ public long[] getLongs() {
+ long[] result = new long[values.length];
+ for (int i = 0; i < values.length; i++) {
+ result[i] = values[i];
+ }
+ return result;
+ }
+
+ @Override
+ public double[] getDoubles() {
+ double[] result = new double[values.length];
+ for (int i = 0; i < values.length; i++) {
+ result[i] = values[i];
+ }
+ return result;
+ }
+
@Override
public Object getObject(int position) {
return getInt(position);
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java
index bbbb0663..ecba9013 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/read/common/block/column/LongColumn.java
@@ -34,6 +34,7 @@ import static org.apache.tsfile.read.common.block.column.ColumnUtil.checkValidRe
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfBooleanArray;
import static org.apache.tsfile.utils.RamUsageEstimator.sizeOfLongArray;
+@SuppressWarnings("java:S3012")
public class LongColumn implements Column {
private static final int INSTANCE_SIZE =
@@ -94,11 +95,25 @@ public class LongColumn implements Column {
return values[position + arrayOffset];
}
+ @Override
+ public double getDouble(int position) {
+ return values[position + arrayOffset];
+ }
+
@Override
public long[] getLongs() {
return values;
}
+ @Override
+ public double[] getDoubles() {
+ double[] doubles = new double[values.length];
+ for (int i = 0; i < values.length; i++) {
+ doubles[i] = values[i];
+ }
+ return doubles;
+ }
+
@Override
public Object getObject(int position) {
return getLong(position);
(tsfile) 01/03: add statistic compatibility
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch statistic_compatibility_pr
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit 5308a4f2e8c1723824ff0a4bea48d42f41cecda5
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Jan 13 11:24:41 2025 +0800
add statistic compatibility
(cherry picked from commit 03c229d21de4322de36c292e3fa411cb1b856500)
(cherry picked from commit c97ec476c812bfa9d0bacc11f33c6077269adc3c)
---
.../file/metadata/statistics/BinaryStatistics.java | 26 +--
.../file/metadata/statistics/BlobStatistics.java | 3 +-
.../metadata/statistics/BooleanStatistics.java | 29 ++--
.../file/metadata/statistics/DoubleStatistics.java | 43 +++--
.../file/metadata/statistics/FloatStatistics.java | 42 ++---
.../metadata/statistics/IntegerStatistics.java | 7 +-
.../file/metadata/statistics/LongStatistics.java | 40 +++--
.../file/metadata/statistics/Statistics.java | 16 +-
.../file/metadata/statistics/StringStatistics.java | 38 +++--
.../file/metadata/statistics/TimeStatistics.java | 3 +-
.../file/metadata/statistics/StatisticsTest.java | 183 +++++++++++++++++++++
11 files changed, 324 insertions(+), 106 deletions(-)
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BinaryStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BinaryStatistics.java
index 34db11b9..9915121d 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BinaryStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BinaryStatistics.java
@@ -125,18 +125,22 @@ public class BinaryStatistics extends Statistics<Binary> {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.TEXT, "long sum"));
}
- @Override
- protected void mergeStatisticsValue(Statistics<Binary> stats) {
- BinaryStatistics stringStats = (BinaryStatistics) stats;
- if (isEmpty) {
- initializeStats(stringStats.getFirstValue(), stringStats.getLastValue());
- isEmpty = false;
+ @SuppressWarnings("rawtypes")
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ if (stats instanceof BinaryStatistics || stats instanceof StringStatistics) {
+ if (isEmpty) {
+ initializeStats(((Binary) stats.getFirstValue()), ((Binary) stats.getLastValue()));
+ isEmpty = false;
+ } else {
+ updateStats(
+ ((Binary) stats.getFirstValue()),
+ ((Binary) stats.getLastValue()),
+ stats.getStartTime(),
+ stats.getEndTime());
+ }
} else {
- updateStats(
- stringStats.getFirstValue(),
- stringStats.getLastValue(),
- stats.getStartTime(),
- stats.getEndTime());
+ throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java
index f8cb18fa..65c7515b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BlobStatistics.java
@@ -110,8 +110,9 @@ public class BlobStatistics extends Statistics<Binary> {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.BLOB, "sum"));
}
+ @SuppressWarnings("rawtypes")
@Override
- protected void mergeStatisticsValue(Statistics<Binary> stats) {
+ protected void mergeStatisticsValue(Statistics stats) {
// do nothing
if (isEmpty) {
isEmpty = false;
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BooleanStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BooleanStatistics.java
index 63e01288..d519a6ca 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BooleanStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/BooleanStatistics.java
@@ -127,8 +127,7 @@ public class BooleanStatistics extends Statistics<Boolean> {
@Override
public double getSumDoubleValue() {
- throw new StatisticsClassException(
- String.format(STATS_UNSUPPORTED_MSG, TSDataType.BOOLEAN, "double sum"));
+ return sumValue;
}
@Override
@@ -137,18 +136,22 @@ public class BooleanStatistics extends Statistics<Boolean> {
}
@Override
- protected void mergeStatisticsValue(Statistics<Boolean> stats) {
- BooleanStatistics boolStats = (BooleanStatistics) stats;
- if (isEmpty) {
- initializeStats(boolStats.getFirstValue(), boolStats.getLastValue(), boolStats.sumValue);
- isEmpty = false;
+ protected void mergeStatisticsValue(Statistics stats) {
+ if (stats instanceof BooleanStatistics) {
+ BooleanStatistics boolStats = (BooleanStatistics) stats;
+ if (isEmpty) {
+ initializeStats(boolStats.getFirstValue(), boolStats.getLastValue(), boolStats.sumValue);
+ isEmpty = false;
+ } else {
+ updateStats(
+ boolStats.getFirstValue(),
+ boolStats.getLastValue(),
+ stats.getStartTime(),
+ stats.getEndTime(),
+ boolStats.sumValue);
+ }
} else {
- updateStats(
- boolStats.getFirstValue(),
- boolStats.getLastValue(),
- stats.getStartTime(),
- stats.getEndTime(),
- boolStats.sumValue);
+ throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DoubleStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DoubleStatistics.java
index d79c4c2c..1d259ca3 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DoubleStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/DoubleStatistics.java
@@ -159,26 +159,33 @@ public class DoubleStatistics extends Statistics<Double> {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.DOUBLE, "long sum"));
}
+ @SuppressWarnings("rawtypes")
@Override
- protected void mergeStatisticsValue(Statistics<Double> stats) {
- DoubleStatistics doubleStats = (DoubleStatistics) stats;
- if (this.isEmpty) {
- initializeStats(
- doubleStats.getMinValue(),
- doubleStats.getMaxValue(),
- doubleStats.getFirstValue(),
- doubleStats.getLastValue(),
- doubleStats.sumValue);
- isEmpty = false;
+ protected void mergeStatisticsValue(Statistics stats) {
+ if (stats instanceof DoubleStatistics
+ || stats instanceof FloatStatistics
+ || stats instanceof IntegerStatistics
+ || stats instanceof LongStatistics) {
+ if (this.isEmpty) {
+ initializeStats(
+ ((Number) stats.getMinValue()).doubleValue(),
+ ((Number) stats.getMaxValue()).doubleValue(),
+ ((Number) stats.getFirstValue()).doubleValue(),
+ ((Number) stats.getLastValue()).doubleValue(),
+ stats.getSumDoubleValue());
+ isEmpty = false;
+ } else {
+ updateStats(
+ ((Number) stats.getMinValue()).doubleValue(),
+ ((Number) stats.getMaxValue()).doubleValue(),
+ ((Number) stats.getFirstValue()).doubleValue(),
+ ((Number) stats.getLastValue()).doubleValue(),
+ stats.getSumDoubleValue(),
+ stats.getStartTime(),
+ stats.getEndTime());
+ }
} else {
- updateStats(
- doubleStats.getMinValue(),
- doubleStats.getMaxValue(),
- doubleStats.getFirstValue(),
- doubleStats.getLastValue(),
- doubleStats.sumValue,
- stats.getStartTime(),
- stats.getEndTime());
+ throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/FloatStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/FloatStatistics.java
index 8df52008..0d9595a8 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/FloatStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/FloatStatistics.java
@@ -150,26 +150,30 @@ public class FloatStatistics extends Statistics<Float> {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.FLOAT, "long sum"));
}
- @Override
- protected void mergeStatisticsValue(Statistics<Float> stats) {
- FloatStatistics floatStats = (FloatStatistics) stats;
- if (isEmpty) {
- initializeStats(
- floatStats.getMinValue(),
- floatStats.getMaxValue(),
- floatStats.getFirstValue(),
- floatStats.getLastValue(),
- floatStats.sumValue);
- isEmpty = false;
+ @SuppressWarnings("rawtypes")
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ if (stats instanceof FloatStatistics || stats instanceof IntegerStatistics) {
+ if (isEmpty) {
+ initializeStats(
+ ((Number) stats.getMinValue()).floatValue(),
+ ((Number) stats.getMaxValue()).floatValue(),
+ ((Number) stats.getFirstValue()).floatValue(),
+ ((Number) stats.getLastValue()).floatValue(),
+ stats.getSumDoubleValue());
+ isEmpty = false;
+ } else {
+ updateStats(
+ ((Number) stats.getMinValue()).floatValue(),
+ ((Number) stats.getMaxValue()).floatValue(),
+ ((Number) stats.getFirstValue()).floatValue(),
+ ((Number) stats.getLastValue()).floatValue(),
+ stats.getSumDoubleValue(),
+ stats.getStartTime(),
+ stats.getEndTime());
+ }
} else {
- updateStats(
- floatStats.getMinValue(),
- floatStats.getMaxValue(),
- floatStats.getFirstValue(),
- floatStats.getLastValue(),
- floatStats.sumValue,
- stats.getStartTime(),
- stats.getEndTime());
+ throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/IntegerStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/IntegerStatistics.java
index 45f17166..804609d9 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/IntegerStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/IntegerStatistics.java
@@ -20,7 +20,6 @@
package org.apache.tsfile.file.metadata.statistics;
import org.apache.tsfile.enums.TSDataType;
-import org.apache.tsfile.exception.filter.StatisticsClassException;
import org.apache.tsfile.utils.RamUsageEstimator;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -142,8 +141,7 @@ public class IntegerStatistics extends Statistics<Integer> {
@Override
public double getSumDoubleValue() {
- throw new StatisticsClassException(
- String.format(STATS_UNSUPPORTED_MSG, TSDataType.INT32, "double sum"));
+ return sumValue;
}
@Override
@@ -151,8 +149,9 @@ public class IntegerStatistics extends Statistics<Integer> {
return sumValue;
}
+ @SuppressWarnings("rawtypes")
@Override
- protected void mergeStatisticsValue(Statistics<Integer> stats) {
+ protected void mergeStatisticsValue(Statistics stats) {
IntegerStatistics intStats = (IntegerStatistics) stats;
if (isEmpty) {
initializeStats(
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/LongStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/LongStatistics.java
index c72668cb..5f362425 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/LongStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/LongStatistics.java
@@ -160,26 +160,30 @@ public class LongStatistics extends Statistics<Long> {
}
}
+ @SuppressWarnings("rawtypes")
@Override
- protected void mergeStatisticsValue(Statistics<Long> stats) {
- LongStatistics longStats = (LongStatistics) stats;
- if (isEmpty) {
- initializeStats(
- longStats.getMinValue(),
- longStats.getMaxValue(),
- longStats.getFirstValue(),
- longStats.getLastValue(),
- longStats.sumValue);
- isEmpty = false;
+ protected void mergeStatisticsValue(Statistics stats) {
+ if (stats instanceof LongStatistics || stats instanceof IntegerStatistics) {
+ if (isEmpty) {
+ initializeStats(
+ ((Number) stats.getMinValue()).longValue(),
+ ((Number) stats.getMaxValue()).longValue(),
+ ((Number) stats.getFirstValue()).longValue(),
+ ((Number) stats.getLastValue()).longValue(),
+ stats.getSumDoubleValue());
+ isEmpty = false;
+ } else {
+ updateStats(
+ ((Number) stats.getMinValue()).longValue(),
+ ((Number) stats.getMaxValue()).longValue(),
+ ((Number) stats.getFirstValue()).longValue(),
+ ((Number) stats.getLastValue()).longValue(),
+ stats.getSumDoubleValue(),
+ stats.getStartTime(),
+ stats.getEndTime());
+ }
} else {
- updateStats(
- longStats.getMinValue(),
- longStats.getMaxValue(),
- longStats.getFirstValue(),
- longStats.getLastValue(),
- longStats.sumValue,
- stats.getStartTime(),
- stats.getEndTime());
+ throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
index 56b3bf7a..3fc0f714 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/Statistics.java
@@ -203,7 +203,7 @@ public abstract class Statistics<T extends Serializable> {
*/
@SuppressWarnings("unchecked")
public void mergeStatistics(Statistics<? extends Serializable> stats) {
- if (this.getClass() == stats.getClass()) {
+ if (this.getClass() == stats.getClass() || canMerge(stats.getType(), this.getType())) {
if (!stats.isEmpty) {
if (stats.startTime < this.startTime) {
this.startTime = stats.startTime;
@@ -213,7 +213,7 @@ public abstract class Statistics<T extends Serializable> {
}
// must be sure no overlap between two statistics
this.count += stats.count;
- mergeStatisticsValue((Statistics<T>) stats);
+ mergeStatisticsValue(stats);
isEmpty = false;
}
} else {
@@ -225,6 +225,13 @@ public abstract class Statistics<T extends Serializable> {
}
}
+ public static boolean canMerge(TSDataType from, TSDataType to) {
+ return to.isCompatible(from)
+ &&
+ // cannot alter from TEXT to STRING because we cannot add statistic to the existing chunks
+ !(from == TSDataType.TEXT && to == TSDataType.STRING);
+ }
+
public void update(long time, boolean value) {
update(time);
updateStats(value);
@@ -315,7 +322,8 @@ public abstract class Statistics<T extends Serializable> {
count += batchSize;
}
- protected abstract void mergeStatisticsValue(Statistics<T> stats);
+ @SuppressWarnings("rawtypes")
+ protected abstract void mergeStatisticsValue(Statistics stats);
public boolean isEmpty() {
return isEmpty;
@@ -393,7 +401,7 @@ public abstract class Statistics<T extends Serializable> {
return endTime;
}
- public long getCount() {
+ public int getCount() {
return count;
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java
index 5997fb9b..5da57af1 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/StringStatistics.java
@@ -143,24 +143,28 @@ public class StringStatistics extends Statistics<Binary> {
String.format(STATS_UNSUPPORTED_MSG, TSDataType.STRING, "long sum"));
}
- @Override
- protected void mergeStatisticsValue(Statistics<Binary> stats) {
- StringStatistics stringStats = (StringStatistics) stats;
- if (isEmpty) {
- initializeStats(
- stringStats.getFirstValue(),
- stringStats.getLastValue(),
- stringStats.getMinValue(),
- stringStats.getMaxValue());
- isEmpty = false;
+ @SuppressWarnings("rawtypes")
+ @Override
+ protected void mergeStatisticsValue(Statistics stats) {
+ if (stats instanceof StringStatistics) {
+ if (isEmpty) {
+ initializeStats(
+ ((Binary) stats.getFirstValue()),
+ ((Binary) stats.getLastValue()),
+ ((Binary) stats.getMinValue()),
+ ((Binary) stats.getMaxValue()));
+ isEmpty = false;
+ } else {
+ updateStats(
+ ((Binary) stats.getFirstValue()),
+ ((Binary) stats.getLastValue()),
+ ((Binary) stats.getMinValue()),
+ ((Binary) stats.getMaxValue()),
+ stats.getStartTime(),
+ stats.getEndTime());
+ }
} else {
- updateStats(
- stringStats.getFirstValue(),
- stringStats.getLastValue(),
- stringStats.getMinValue(),
- stringStats.getMaxValue(),
- stats.getStartTime(),
- stats.getEndTime());
+ throw new StatisticsClassException(this.getClass(), stats.getClass());
}
}
diff --git a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimeStatistics.java b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimeStatistics.java
index 48fcb329..db12ac4b 100644
--- a/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimeStatistics.java
+++ b/java/tsfile/src/main/java/org/apache/tsfile/file/metadata/statistics/TimeStatistics.java
@@ -119,8 +119,9 @@ public class TimeStatistics extends Statistics<Long> {
throw new StatisticsClassException(String.format(STATS_UNSUPPORTED_MSG, TIME, UPDATE_STATS));
}
+ @SuppressWarnings("rawtypes")
@Override
- protected void mergeStatisticsValue(Statistics<Long> stats) {}
+ protected void mergeStatisticsValue(Statistics stats) {}
@Override
public int serializeStats(OutputStream outputStream) {
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java
new file mode 100644
index 00000000..06aeca38
--- /dev/null
+++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/StatisticsTest.java
@@ -0,0 +1,183 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.tsfile.file.metadata.statistics;
+
+import org.apache.tsfile.common.conf.TSFileConfig;
+import org.apache.tsfile.enums.TSDataType;
+import org.apache.tsfile.utils.Binary;
+
+import org.junit.Test;
+
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+@SuppressWarnings({"unchecked", "rawtypes"})
+public class StatisticsTest {
+
+ @Test
+ public void testCrossTypeMerge() {
+ Set<TSDataType> dataTypes = new HashSet<>();
+ Collections.addAll(dataTypes, TSDataType.values());
+ dataTypes.remove(TSDataType.VECTOR);
+ dataTypes.remove(TSDataType.UNKNOWN);
+
+ for (TSDataType from : dataTypes) {
+ for (TSDataType to : dataTypes) {
+ Statistics fromStatistics = genStatistics(from, 0);
+ Statistics toStatistics = genStatistics(to, 1);
+ if (Statistics.canMerge(from, to)) {
+ toStatistics.mergeStatistics(fromStatistics);
+ checkStatistics(toStatistics, 0, 1, 1.0);
+ } else {
+ try {
+ toStatistics.mergeStatistics(fromStatistics);
+ fail("Expected MergeException");
+ } catch (Exception e) {
+ assertEquals(
+ String.format(
+ "Statistics classes mismatched: %s vs. %s",
+ toStatistics.getClass(), fromStatistics.getClass()),
+ e.getMessage());
+ }
+ checkStatistics(toStatistics, 1, 1, 1.0);
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private static void checkStatistics(Statistics statistics, int min, int max, double sum) {
+ assertEquals(min, statistics.getStartTime());
+ assertEquals(max, statistics.getEndTime());
+ switch (statistics.getType()) {
+ case INT32:
+ case INT64:
+ case FLOAT:
+ case DOUBLE:
+ case TIMESTAMP:
+ case DATE:
+ assertEquals(min, ((Number) statistics.getMinValue()).intValue());
+ assertEquals(max, ((Number) statistics.getMaxValue()).intValue());
+ assertEquals(min, ((Number) statistics.getFirstValue()).intValue());
+ assertEquals(max, ((Number) statistics.getLastValue()).intValue());
+ assertEquals(sum, statistics.getSumDoubleValue(), 0.001);
+ break;
+ case BOOLEAN:
+ assertEquals(min % 2 == 1, statistics.getFirstValue());
+ assertEquals(max % 2 == 1, statistics.getLastValue());
+ assertEquals(sum, statistics.getSumDoubleValue(), 0.001);
+ break;
+ case TEXT:
+ assertEquals(
+ new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET),
+ statistics.getFirstValue());
+ assertEquals(
+ new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET),
+ statistics.getLastValue());
+ break;
+ case STRING:
+ assertEquals(
+ new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET), statistics.getMinValue());
+ assertEquals(
+ new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET), statistics.getMaxValue());
+ assertEquals(
+ new Binary(String.valueOf(min), TSFileConfig.STRING_CHARSET),
+ statistics.getFirstValue());
+ assertEquals(
+ new Binary(String.valueOf(max), TSFileConfig.STRING_CHARSET),
+ statistics.getLastValue());
+ break;
+ case BLOB:
+ break;
+ default:
+ throw new IllegalArgumentException(statistics.getType().toString());
+ }
+ }
+
+ private static Statistics genStatistics(TSDataType dataType, int val) {
+ Statistics result;
+ switch (dataType) {
+ case INT32:
+ IntegerStatistics intStat = new IntegerStatistics();
+ intStat.initializeStats(val, val, val, val, val);
+ result = intStat;
+ break;
+ case INT64:
+ LongStatistics longStat = new LongStatistics();
+ longStat.initializeStats(val, val, val, val, val);
+ result = longStat;
+ break;
+ case FLOAT:
+ FloatStatistics floatStat = new FloatStatistics();
+ floatStat.initializeStats(val, val, val, val, val);
+ result = floatStat;
+ break;
+ case DOUBLE:
+ DoubleStatistics doubleStat = new DoubleStatistics();
+ doubleStat.initializeStats(val, val, val, val, val);
+ result = doubleStat;
+ break;
+ case TEXT:
+ BinaryStatistics binaryStat = new BinaryStatistics();
+ binaryStat.initializeStats(
+ new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET),
+ new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET));
+ result = binaryStat;
+ break;
+ case STRING:
+ StringStatistics stringStat = new StringStatistics();
+ stringStat.initializeStats(
+ new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET),
+ new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET),
+ new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET),
+ new Binary(String.valueOf(val), TSFileConfig.STRING_CHARSET));
+ result = stringStat;
+ break;
+ case BOOLEAN:
+ BooleanStatistics boolStat = new BooleanStatistics();
+ boolStat.initializeStats(val % 2 == 1, val % 2 == 1, val % 2 == 1 ? 1 : 0);
+ result = boolStat;
+ break;
+ case BLOB:
+ BlobStatistics blobStat = new BlobStatistics();
+ result = blobStat;
+ break;
+ case DATE:
+ DateStatistics dateStat = new DateStatistics();
+ dateStat.initializeStats(val, val, val, val, val);
+ result = dateStat;
+ break;
+ case TIMESTAMP:
+ TimestampStatistics timestampStat = new TimestampStatistics();
+ timestampStat.initializeStats(val, val, val, val, val);
+ result = timestampStat;
+ break;
+ default:
+ throw new IllegalArgumentException(dataType.toString());
+ }
+ result.setStartTime(val);
+ result.setEndTime(val);
+ result.setEmpty(false);
+ return result;
+ }
+}
(tsfile) 02/03: fix test
Posted by ji...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch statistic_compatibility_pr
in repository https://gitbox.apache.org/repos/asf/tsfile.git
commit f65a1f6e6f52a03439acb4a56ccd8d23aa4feb90
Author: Tian Jiang <jt...@163.com>
AuthorDate: Mon Jan 13 12:06:27 2025 +0800
fix test
(cherry picked from commit 92a06d16863f90eab61d270b9bd5d7e4d1e89cf1)
---
.../metadata/statistics/LongStatisticsTest.java | 22 ----------------------
1 file changed, 22 deletions(-)
diff --git a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/LongStatisticsTest.java b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/LongStatisticsTest.java
index 47a34699..39b95e15 100644
--- a/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/LongStatisticsTest.java
+++ b/java/tsfile/src/test/java/org/apache/tsfile/file/metadata/statistics/LongStatisticsTest.java
@@ -18,14 +18,11 @@
*/
package org.apache.tsfile.file.metadata.statistics;
-import org.apache.tsfile.exception.filter.StatisticsClassException;
-
import org.junit.Test;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
public class LongStatisticsTest {
@@ -78,25 +75,6 @@ public class LongStatisticsTest {
assertEquals(1, (long) longStats3.getFirstValue());
assertEquals(max2, (long) longStats3.getLastValue());
- // Test mismatch
- IntegerStatistics intStats5 = new IntegerStatistics();
- intStats5.updateStats(-10000);
- try {
- longStats3.mergeStatistics(intStats5);
- } catch (StatisticsClassException e) {
- // that's true route
- } catch (Exception e) {
- fail();
- }
-
- assertEquals(max2, (long) longStats3.getMaxValue());
- // if not merge, the min value will not be changed by smaller value in
- // intStats5
- assertEquals(1, (long) longStats3.getMinValue());
- assertEquals(max2 + max1 + 1, (long) longStats3.getSumDoubleValue());
- assertEquals(1, (long) longStats3.getFirstValue());
- assertEquals(max2, (long) longStats3.getLastValue());
-
// Unseq Merge
LongStatistics longStats4 = new LongStatistics();
longStats4.setStartTime(0);