You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2020/06/08 03:13:48 UTC
[incubator-iotdb] 02/04: fix sort bug
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch nvmlogging
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit cc6d7c15115f1543ddbaa9ab8e8efb8b9461fceb
Author: mdf369 <95...@qq.com>
AuthorDate: Sat Jun 6 01:27:03 2020 +0800
fix sort bug
---
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../iotdb/db/engine/flush/MemTableFlushTask.java | 5 -----
.../iotdb/db/query/executor/AggregationExecutor.java | 1 -
.../iotdb/db/query/reader/series/SeriesReader.java | 1 -
.../iotdb/db/utils/datastructure/AbstractTVList.java | 11 ++++-------
.../iotdb/db/utils/datastructure/NVMBinaryTVList.java | 9 ++++-----
.../iotdb/db/utils/datastructure/NVMBooleanTVList.java | 5 +++--
.../iotdb/db/utils/datastructure/NVMDoubleTVList.java | 5 +++--
.../iotdb/db/utils/datastructure/NVMFloatTVList.java | 5 +++--
.../iotdb/db/utils/datastructure/NVMIntTVList.java | 5 +++--
.../iotdb/db/utils/datastructure/NVMLongTVList.java | 5 +++--
.../apache/iotdb/db/utils/datastructure/NVMTVList.java | 18 ++++++++++++++++--
.../apache/iotdb/db/utils/datastructure/TVList.java | 12 ++++++++++++
.../org/apache/iotdb/tsfile/read/common/BatchData.java | 2 +-
.../iotdb/tsfile/read/reader/chunk/ChunkReader.java | 1 -
15 files changed, 53 insertions(+), 34 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index f0c3b04..6741805 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -190,7 +190,7 @@ public class IoTDBConfig {
/**
* Is NVM enable.
*/
- private boolean enableNVM = false;
+ private boolean enableNVM = true;
/**
* NVM directory.
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
index 8c9983c..77ec37c 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/flush/MemTableFlushTask.java
@@ -80,11 +80,6 @@ public class MemTableFlushTask {
IWritableMemChunk series = memTable.getMemTableMap().get(deviceId).get(measurementId);
MeasurementSchema desc = series.getSchema();
AbstractTVList tvList = series.getSortedTVList();
- System.out.println("flush size:" + tvList.size());
- for (int i = 0; i < tvList.size(); i++) {
- System.out.print("<" + tvList.getTime(i) + ":" + tvList.getInt(i) + "> ");
- }
- System.out.println();
sortTime += System.currentTimeMillis() - startTime;
encodingTaskQueue.add(new Pair<>(tvList, desc));
// register active time series to the ActiveTimeSeriesCounter
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
index fae41c4..04e80f2 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/AggregationExecutor.java
@@ -231,7 +231,6 @@ public class AggregationExecutor {
continue;
}
BatchData nextOverlappedPageData = seriesReader.nextPage();
- System.out.println("read:" + nextOverlappedPageData.count);
for (int i = 0; i < aggregateResultList.size(); i++) {
if (!isCalculatedArray[i]) {
AggregateResult aggregateResult = aggregateResultList.get(i);
diff --git a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
index b1e2eb6..816be29 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/reader/series/SeriesReader.java
@@ -477,7 +477,6 @@ class SeriesReader {
* get current first point in mergeReader, this maybe overlapped latter
*/
TimeValuePair timeValuePair = mergeReader.currentTimeValuePair();
- System.out.println("read:" + timeValuePair);
if (timeValuePair.getTimestamp() > currentPageEndTime) {
break;
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AbstractTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AbstractTVList.java
index d8728f3..4e84e8e 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AbstractTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/AbstractTVList.java
@@ -136,6 +136,8 @@ public abstract class AbstractTVList {
protected abstract void set(int src, int dest);
+ protected abstract void setForSort(int src, int dest);
+
protected abstract void setFromSorted(int src, int dest);
protected abstract void setToSorted(int src, int dest);
@@ -234,11 +236,7 @@ public abstract class AbstractTVList {
this.timeOffset = timeOffset;
}
- protected int compare(int idx1, int idx2) {
- long t1 = getTime(idx1);
- long t2 = getTime(idx2);
- return Long.compare(t1, t2);
- }
+ protected abstract int compare(int idx1, int idx2);
protected abstract void saveAsPivot(int pos);
@@ -283,8 +281,7 @@ public abstract class AbstractTVList {
*/
int n = start - left; // The number of elements to move
for (int i = n; i >= 1; i--) {
- set(left + i - 1, left + i);
- setToSorted(left + i - 1, left + i);
+ setForSort(left + i - 1, left + i);
}
setPivotTo(left);
}
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java
index 88c37fc..9087abd 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBinaryTVList.java
@@ -2,11 +2,9 @@ package org.apache.iotdb.db.utils.datastructure;
import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
-import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool;
import org.apache.iotdb.db.nvm.space.NVMBinaryDataSpace;
import org.apache.iotdb.db.nvm.space.NVMDataSpace;
-import org.apache.iotdb.db.nvm.space.NVMSpaceManager;
import org.apache.iotdb.db.nvm.space.NVMSpaceMetadataManager;
import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -92,7 +90,7 @@ public class NVMBinaryTVList extends NVMTVList {
}
@Override
- protected void initTempArrays() {
+ protected void initSortedAndTempArrays() {
if (sortedTimestamps == null || sortedTimestamps.length < size) {
sortedTimestamps = (long[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.INT64, size);
@@ -163,7 +161,8 @@ public class NVMBinaryTVList extends NVMTVList {
@Override
protected void setFromSorted(int src, int dest) {
- setForSort(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ setForSort(dest, sortedTimestamps[src / ARRAY_SIZE][src % ARRAY_SIZE],
+ sortedValues[src / ARRAY_SIZE][src % ARRAY_SIZE]);
}
@Override
@@ -175,7 +174,7 @@ public class NVMBinaryTVList extends NVMTVList {
@Override
protected void setValueForSort(int arrayIndex, int elementIndex, Object value) {
- tempValuesForSort[arrayIndex][elementIndex] = (Binary) value;
+ tempValuesForSort[arrayIndex * ARRAY_SIZE + elementIndex][0] = (Binary) value;
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBooleanTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBooleanTVList.java
index 726fb1e..f6e7902 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBooleanTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMBooleanTVList.java
@@ -47,7 +47,7 @@ public class NVMBooleanTVList extends NVMTVList {
}
@Override
- protected void initTempArrays() {
+ protected void initSortedAndTempArrays() {
if (sortedTimestamps == null || sortedTimestamps.length < size) {
sortedTimestamps = (long[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.INT64, size);
@@ -118,7 +118,8 @@ public class NVMBooleanTVList extends NVMTVList {
@Override
protected void setFromSorted(int src, int dest) {
- setForSort(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ setForSort(dest, sortedTimestamps[src / ARRAY_SIZE][src % ARRAY_SIZE],
+ sortedValues[src / ARRAY_SIZE][src % ARRAY_SIZE]);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java
index c8bdee4..6e12391 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMDoubleTVList.java
@@ -48,7 +48,7 @@ public class NVMDoubleTVList extends NVMTVList {
}
@Override
- protected void initTempArrays() {
+ protected void initSortedAndTempArrays() {
if (sortedTimestamps == null || sortedTimestamps.length < size) {
sortedTimestamps = (long[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.INT64, size);
@@ -119,7 +119,8 @@ public class NVMDoubleTVList extends NVMTVList {
@Override
protected void setFromSorted(int src, int dest) {
- setForSort(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ setForSort(dest, sortedTimestamps[src / ARRAY_SIZE][src % ARRAY_SIZE],
+ sortedValues[src / ARRAY_SIZE][src % ARRAY_SIZE]);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
index d72e05a..fa7b517 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMFloatTVList.java
@@ -47,7 +47,7 @@ public class NVMFloatTVList extends NVMTVList {
}
@Override
- protected void initTempArrays() {
+ protected void initSortedAndTempArrays() {
if (sortedTimestamps == null || sortedTimestamps.length < size) {
sortedTimestamps = (long[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.INT64, size);
@@ -118,7 +118,8 @@ public class NVMFloatTVList extends NVMTVList {
@Override
protected void setFromSorted(int src, int dest) {
- setForSort(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ setForSort(dest, sortedTimestamps[src / ARRAY_SIZE][src % ARRAY_SIZE],
+ sortedValues[src / ARRAY_SIZE][src % ARRAY_SIZE]);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMIntTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMIntTVList.java
index 13e5e0a..67b7e77 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMIntTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMIntTVList.java
@@ -47,7 +47,7 @@ public class NVMIntTVList extends NVMTVList {
}
@Override
- protected void initTempArrays() {
+ protected void initSortedAndTempArrays() {
if (sortedTimestamps == null || sortedTimestamps.length < size) {
sortedTimestamps = (long[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.INT64, size);
@@ -118,7 +118,8 @@ public class NVMIntTVList extends NVMTVList {
@Override
protected void setFromSorted(int src, int dest) {
- setForSort(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ setForSort(dest, sortedTimestamps[src / ARRAY_SIZE][src % ARRAY_SIZE],
+ sortedValues[src / ARRAY_SIZE][src % ARRAY_SIZE]);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMLongTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMLongTVList.java
index 98823be..774890a 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMLongTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMLongTVList.java
@@ -47,7 +47,7 @@ public class NVMLongTVList extends NVMTVList {
}
@Override
- protected void initTempArrays() {
+ protected void initSortedAndTempArrays() {
if (sortedTimestamps == null || sortedTimestamps.length < size) {
sortedTimestamps = (long[][]) PrimitiveArrayPool
.getInstance().getDataListsByType(TSDataType.INT64, size);
@@ -118,7 +118,8 @@ public class NVMLongTVList extends NVMTVList {
@Override
protected void setFromSorted(int src, int dest) {
- setForSort(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ setForSort(dest, sortedTimestamps[src / ARRAY_SIZE][src % ARRAY_SIZE],
+ sortedValues[src / ARRAY_SIZE][src % ARRAY_SIZE]);
}
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
index 6e4d2d1..37eea2f 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/NVMTVList.java
@@ -241,7 +241,7 @@ public abstract class NVMTVList extends AbstractTVList {
@Override
public void sort() {
- initTempArrays();
+ initSortedAndTempArrays();
copyTVToTempArrays();
@@ -255,7 +255,7 @@ public abstract class NVMTVList extends AbstractTVList {
sorted = true;
}
- protected abstract void initTempArrays();
+ protected abstract void initSortedAndTempArrays();
protected abstract void copyTVToTempArrays();
@@ -296,6 +296,13 @@ public abstract class NVMTVList extends AbstractTVList {
}
@Override
+ protected void setForSort(int src, int dest) {
+ long srcT = getTimeForSort(src);
+ Object srcV = getValueForSort(src);
+ setForSort(dest, srcT, srcV);
+ }
+
+ @Override
protected void setForSort(int index, long timestamp, Object value) {
if (index >= size) {
throw new ArrayIndexOutOfBoundsException(index);
@@ -309,6 +316,13 @@ public abstract class NVMTVList extends AbstractTVList {
protected abstract void setValueForSort(int arrayIndex, int elementIndex, Object value);
@Override
+ protected int compare(int idx1, int idx2) {
+ long t1 = getTimeForSort(idx1);
+ long t2 = getTimeForSort(idx2);
+ return Long.compare(t1, t2);
+ }
+
+ @Override
protected void saveAsPivot(int pos) {
pivotTime = getTimeForSort(pos);
pivotValue = getValueForSort(pos);
diff --git a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
index f9a3df7..4b7cfd3 100644
--- a/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/utils/datastructure/TVList.java
@@ -108,6 +108,18 @@ public abstract class TVList extends AbstractTVList {
}
@Override
+ protected int compare(int idx1, int idx2) {
+ long t1 = getTime(idx1);
+ long t2 = getTime(idx2);
+ return Long.compare(t1, t2);
+ }
+
+ @Override
+ protected void setForSort(int src, int dest) {
+ set(src, dest);
+ }
+
+ @Override
protected void clearSortedTime() {
if (sortedTimestamps != null) {
for (long[] dataArray : sortedTimestamps) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
index da9dd83..a893c1b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/BatchData.java
@@ -68,7 +68,7 @@ public class BatchData implements Serializable {
private int writeCurArrayIndex;
// the insert timestamp number of timeRet
- public int count;
+ private int count;
private List<long[]> timeRet;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
index 4289952..aa334d8 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/reader/chunk/ChunkReader.java
@@ -93,7 +93,6 @@ public class ChunkReader implements IChunkReader {
// deserialize a PageHeader from chunkDataBuffer
PageHeader pageHeader = isFromOldTsFile ? HeaderUtils.deserializePageHeaderV1(chunkDataBuffer, chunkHeader.getDataType()) :
PageHeader.deserializeFrom(chunkDataBuffer, chunkHeader.getDataType());
- System.out.println("statistics:" + pageHeader.getStatistics());
// if the current page satisfies
if (pageSatisfied(pageHeader)) {
pageReaderList.add(constructPageReaderForNextPage(pageHeader));