You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xi...@apache.org on 2021/07/26 01:49:32 UTC
[iotdb] branch clusterDescBug created (now 8dbcd2f)
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a change to branch clusterDescBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 8dbcd2f Fix commit conflict
This branch includes the following new commits:
new a741410 fix a serve bug of BatchData serialization and deserialization which will lead to wrong result of query desc
new 8dbcd2f Fix commit conflict
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/02: fix a serve bug of BatchData serialization and
deserialization which will lead to wrong result of query desc
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch clusterDescBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a741410f6bd1c6c642b10989f431655bf84b403a
Author: lta <li...@163.com>
AuthorDate: Fri Jun 25 01:31:05 2021 +0800
fix a serve bug of BatchData serialization and deserialization which will lead to wrong result of query desc
---
.../org/apache/iotdb/db/utils/SerializeUtils.java | 5 +++-
.../apache/iotdb/tsfile/read/common/BatchData.java | 33 ++++++++++++++++++++++
.../tsfile/read/common/DescReadBatchData.java | 5 +++-
.../tsfile/read/common/DescReadWriteBatchData.java | 1 +
4 files changed, 42 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/SerializeUtils.java b/server/src/main/java/org/apache/iotdb/db/utils/SerializeUtils.java
index cb881e0..c2ccae9 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/SerializeUtils.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/SerializeUtils.java
@@ -23,6 +23,7 @@ import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.common.BatchData.BatchDataType;
import org.apache.iotdb.tsfile.read.filter.basic.Filter;
import org.apache.iotdb.tsfile.utils.Binary;
import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
@@ -123,6 +124,7 @@ public class SerializeUtils {
TSDataType dataType = batchData.getDataType();
outputStream.writeInt(length);
outputStream.write(dataType.ordinal());
+ outputStream.write(batchData.getBatchDataType().ordinal());
switch (dataType) {
case BOOLEAN:
for (int i = 0; i < length; i++) {
@@ -175,7 +177,7 @@ public class SerializeUtils {
int length = buffer.getInt();
TSDataType dataType = TSDataType.values()[buffer.get()];
- BatchData batchData = new BatchData(dataType);
+ BatchData batchData = BatchDataType.deserialize(buffer.get(), dataType);
switch (dataType) {
case INT32:
for (int i = 0; i < length; i++) {
@@ -212,6 +214,7 @@ public class SerializeUtils {
}
break;
}
+ batchData.resetBatchData();
return batchData;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
index c2a7d3b..ab07c0a 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
@@ -57,6 +57,8 @@ public class BatchData {
protected TSDataType dataType;
+ protected BatchDataType batchDataType = BatchDataType.Ordinary;
+
// outer list index for read
protected int readCurListIndex;
// inner array index for read
@@ -162,6 +164,10 @@ public class BatchData {
return dataType;
}
+ public BatchDataType getBatchDataType() {
+ return batchDataType;
+ }
+
/**
* initialize batch data.
*
@@ -599,4 +605,31 @@ public class BatchData {
public BatchData flip() {
return this;
}
+
+ public enum BatchDataType {
+ Ordinary,
+ DescRead,
+ DescReadWrite;
+
+ BatchDataType() {}
+
+ /**
+ * give an integer to return a BatchType type.
+ *
+ * @param type -param to judge enum type
+ * @return -enum type
+ */
+ public static BatchData deserialize(byte type, TSDataType dataType) {
+ switch (type) {
+ case 0:
+ return new BatchData(dataType);
+ case 1:
+ return new DescReadBatchData(dataType);
+ case 2:
+ return new DescReadWriteBatchData(dataType);
+ default:
+ throw new IllegalArgumentException("Invalid input: " + type);
+ }
+ }
+ }
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadBatchData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadBatchData.java
index 9f7d539..ebfc204 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadBatchData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadBatchData.java
@@ -28,10 +28,13 @@ import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
*/
public class DescReadBatchData extends BatchData {
- public DescReadBatchData() {}
+ public DescReadBatchData() {
+ batchDataType = BatchDataType.DescRead;
+ }
public DescReadBatchData(TSDataType dataType) {
super(dataType);
+ batchDataType = BatchDataType.DescRead;
}
@Override
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadWriteBatchData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadWriteBatchData.java
index c299534..ed6e6c2 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadWriteBatchData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/DescReadWriteBatchData.java
@@ -34,6 +34,7 @@ public class DescReadWriteBatchData extends DescReadBatchData {
public DescReadWriteBatchData(TSDataType dataType) {
super();
+ this.batchDataType = BatchDataType.DescReadWrite;
this.dataType = dataType;
this.readCurListIndex = 0;
this.readCurArrayIndex = 0;
[iotdb] 02/02: Fix commit conflict
Posted by xi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xiangweiwei pushed a commit to branch clusterDescBug
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8dbcd2f05e303055219609636fd24e8df021966d
Author: lta <li...@163.com>
AuthorDate: Wed Jun 23 10:59:46 2021 +0800
Fix commit conflict
---
.../iotdb/cluster/query/reader/ClusterReaderFactory.java | 12 ++++++++++--
...dMergeReader.java => ManagedDescPriorityMergeReader.java} | 8 +++++---
...nagedMergeReader.java => ManagedPriorityMergeReader.java} | 4 ++--
.../iotdb/cluster/query/reader/mult/MultBatchReader.java | 5 +++--
4 files changed, 20 insertions(+), 9 deletions(-)
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
index 64a2e3b..3555af2 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ClusterReaderFactory.java
@@ -62,6 +62,7 @@ import org.apache.iotdb.db.query.reader.series.SeriesRawDataBatchReader;
import org.apache.iotdb.db.query.reader.series.SeriesRawDataPointReader;
import org.apache.iotdb.db.query.reader.series.SeriesReader;
import org.apache.iotdb.db.query.reader.series.SeriesReaderByTimestamp;
+import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader;
import org.apache.iotdb.db.utils.SerializeUtils;
import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -364,7 +365,12 @@ public class ClusterReaderFactory {
metaGroupMember.getName(),
path,
partitionGroups.size());
- ManagedMergeReader mergeReader = new ManagedMergeReader(dataType);
+ PriorityMergeReader mergeReader;
+ if (ascending) {
+ mergeReader = new ManagedPriorityMergeReader(dataType);
+ } else {
+ mergeReader = new ManagedDescPriorityMergeReader(dataType);
+ }
try {
// build a reader for each group and merge them
for (PartitionGroup partitionGroup : partitionGroups) {
@@ -383,7 +389,9 @@ public class ClusterReaderFactory {
} catch (IOException | QueryProcessException e) {
throw new StorageEngineException(e);
}
- return mergeReader;
+ // The instance of merge reader is either ManagedPriorityMergeReader or
+ // ManagedDescPriorityMergeReader, which is safe to cast type.
+ return (ManagedSeriesReader) mergeReader;
}
/**
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedMergeReader.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedDescPriorityMergeReader.java
similarity index 88%
copy from cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedMergeReader.java
copy to cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedDescPriorityMergeReader.java
index e54dede..653dea4 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedMergeReader.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedDescPriorityMergeReader.java
@@ -20,16 +20,18 @@
package org.apache.iotdb.cluster.query.reader;
import org.apache.iotdb.db.query.reader.series.ManagedSeriesReader;
-import org.apache.iotdb.db.query.reader.universal.PriorityMergeReader;
+import org.apache.iotdb.db.query.reader.universal.DescPriorityMergeReader;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.TimeValuePair;
import org.apache.iotdb.tsfile.read.common.BatchData;
+import org.apache.iotdb.tsfile.read.reader.IPointReader;
import java.io.IOException;
import java.util.NoSuchElementException;
@SuppressWarnings("common-java:DuplicatedBlocks")
-public class ManagedMergeReader extends PriorityMergeReader implements ManagedSeriesReader {
+public class ManagedDescPriorityMergeReader extends DescPriorityMergeReader
+ implements ManagedSeriesReader, IPointReader {
private static final int BATCH_SIZE = 4096;
@@ -39,7 +41,7 @@ public class ManagedMergeReader extends PriorityMergeReader implements ManagedSe
private BatchData batchData;
private TSDataType dataType;
- public ManagedMergeReader(TSDataType dataType) {
+ public ManagedDescPriorityMergeReader(TSDataType dataType) {
this.dataType = dataType;
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedMergeReader.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedPriorityMergeReader.java
similarity index 94%
rename from cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedMergeReader.java
rename to cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedPriorityMergeReader.java
index e54dede..e57f4d9 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedMergeReader.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/ManagedPriorityMergeReader.java
@@ -29,7 +29,7 @@ import java.io.IOException;
import java.util.NoSuchElementException;
@SuppressWarnings("common-java:DuplicatedBlocks")
-public class ManagedMergeReader extends PriorityMergeReader implements ManagedSeriesReader {
+public class ManagedPriorityMergeReader extends PriorityMergeReader implements ManagedSeriesReader {
private static final int BATCH_SIZE = 4096;
@@ -39,7 +39,7 @@ public class ManagedMergeReader extends PriorityMergeReader implements ManagedSe
private BatchData batchData;
private TSDataType dataType;
- public ManagedMergeReader(TSDataType dataType) {
+ public ManagedPriorityMergeReader(TSDataType dataType) {
this.dataType = dataType;
}
diff --git a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/mult/MultBatchReader.java b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/mult/MultBatchReader.java
index dd7f21b..dc47deb 100644
--- a/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/mult/MultBatchReader.java
+++ b/cluster/src/main/java/org/apache/iotdb/cluster/query/reader/mult/MultBatchReader.java
@@ -41,7 +41,7 @@ public class MultBatchReader implements IMultBatchReader {
@Override
public boolean hasNextBatch() throws IOException {
for (IBatchReader reader : pathBatchReaders.values()) {
- if (reader.hasNextBatch()) {
+ if (reader != null && reader.hasNextBatch()) {
return true;
}
}
@@ -50,7 +50,8 @@ public class MultBatchReader implements IMultBatchReader {
@Override
public boolean hasNextBatch(String fullPath) throws IOException {
- return pathBatchReaders.get(fullPath).hasNextBatch();
+ IBatchReader reader = pathBatchReaders.get(fullPath);
+ return reader != null && reader.hasNextBatch();
}
@Override