You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@iotdb.apache.org by "ycycse (via GitHub)" <gi...@apache.org> on 2023/04/28 09:56:04 UTC
[GitHub] [iotdb] ycycse opened a new pull request, #9746: [IOTDB-5765] Support Order By Expression [BE Part]
ycycse opened a new pull request, #9746:
URL: https://github.com/apache/iotdb/pull/9746
The be part of order by expression which modify sortOperator to support external sort.
Design doc: https://apache-iotdb.feishu.cn/docx/T1msdaV6OoTLczxzQmQccdYunif
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186664589
##########
integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java:
##########
@@ -99,6 +99,7 @@ public class IoTDBOrderByIT {
@BeforeClass
public static void setUp() throws Exception {
+ EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186675625
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
+ }
+ }
+
@Override
public boolean hasNext() throws Exception {
- return inputOperator.hasNextWithTimer() || cachedData != null;
+ return inputOperator.hasNextWithTimer()
+ || (!diskSpiller.hasSpilledData() && curRow != cachedData.size())
+ || (diskSpiller.hasSpilledData() && hasMoreData());
}
@Override
public void close() throws Exception {
+ cachedData = null;
+ clear();
inputOperator.close();
}
@Override
public boolean isFinished() throws Exception {
- return cachedData == null;
+ return inputOperator.isFinished()
+ && (cachedData == null || cachedData.size() == curRow)
+ && ((diskSpiller.hasSpilledData() && !hasMoreData()) || !diskSpiller.hasSpilledData());
Review Comment:
done. The method was changed to `return inputOperator.isFinished()&&!hasNext()`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186668500
##########
server/src/main/java/org/apache/iotdb/db/tools/MemoryReader.java:
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+
+import java.util.List;
+
+public class MemoryReader implements SortReader {
+
+ // all the data in MemoryReader lies in memory
+ private final List<MergeSortKey> cachedData;
+ private int rowIndex;
+
+ public MemoryReader(List<MergeSortKey> cachedTsBlock) {
+ this.cachedData = cachedTsBlock;
+ this.rowIndex = 0;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = cachedData.get(rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ @Override
+ public boolean hasNext() throws IoTDBException {
+ return cachedData != null && rowIndex != cachedData.size();
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186675591
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186655978
##########
integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java:
##########
@@ -99,6 +99,7 @@ public class IoTDBOrderByIT {
@BeforeClass
public static void setUp() throws Exception {
+ EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(10 * 1024 * 1024L);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186670660
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
+
+ public FileSpillerReader(String fileName, SortBufferManager sortBufferManager, TsBlockSerde serde)
+ throws FileNotFoundException {
+ this.fileInputStream = new FileInputStream(fileName);
+ this.cacheBlocks = new ArrayList<>();
+ this.rowIndex = 0;
+ this.sortBufferManager = sortBufferManager;
+ this.serde = serde;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = new MergeSortKey(cacheBlocks.get(tsBlockIndex), rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ private boolean readTsBlockFromFile() throws IoTDBException {
+ long bufferSize = sortBufferManager.getReaderBufferAvailable();
+ cacheBlocks.clear();
+ while (bufferSize >= DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES) {
+ long size = read();
+ if (size == -1) break;
+ bufferSize -= size;
+ }
+
+ if (cacheBlocks.isEmpty()) {
+ return false;
+ }
+ rowIndex = 0;
+ tsBlockIndex = 0;
+ return true;
+ }
+
+ private long read() throws IoTDBException {
+ byte[] bytes = new byte[4];
+ try {
+ int readLen = fileInputStream.read(bytes);
+ if (readLen == -1) {
+ return -1;
+ }
+ int capacity = BytesUtils.bytesToInt(bytes);
+ byte[] tsBlockBytes = ReadWriteIOUtils.readBytes(fileInputStream, capacity);
+ ByteBuffer buffer = ByteBuffer.wrap(tsBlockBytes);
+ TsBlock cachedTsBlock = serde.deserialize(buffer);
+ cacheBlocks.add(cachedTsBlock);
+ return cachedTsBlock.getRetainedSizeInBytes();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ @Override
+ public boolean hasNext() throws IoTDBException {
+
+ if (isEnd) return false;
+
+ if (cacheBlocks.isEmpty()
+ || (rowIndex == cacheBlocks.get(tsBlockIndex).getPositionCount()
+ && tsBlockIndex == cacheBlocks.size() - 1)) {
+ boolean hasData = readTsBlockFromFile();
+ if (!hasData) {
+ isEnd = true;
+ return false;
+ }
+ return true;
+ }
+
+ if (rowIndex < cacheBlocks.get(tsBlockIndex).getPositionCount()) {
+ return true;
+ }
+
+ tsBlockIndex++;
+ rowIndex = 0;
+ return true;
+ }
+
+ @Override
+ public void close() throws IoTDBException {
+ try {
+ fileInputStream.close();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186671115
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186671648
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] JackieTien97 commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "JackieTien97 (via GitHub)" <gi...@apache.org>.
JackieTien97 commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1185733751
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
Review Comment:
no need to save sortBufferManager here, you only use `ReaderBufferAvailable` in SortBufferManager.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674236
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
+ }
+ }
+
@Override
public boolean hasNext() throws Exception {
- return inputOperator.hasNextWithTimer() || cachedData != null;
+ return inputOperator.hasNextWithTimer()
+ || (!diskSpiller.hasSpilledData() && curRow != cachedData.size())
+ || (diskSpiller.hasSpilledData() && hasMoreData());
}
@Override
public void close() throws Exception {
+ cachedData = null;
+ clear();
inputOperator.close();
}
@Override
public boolean isFinished() throws Exception {
- return cachedData == null;
+ return inputOperator.isFinished()
+ && (cachedData == null || cachedData.size() == curRow)
+ && ((diskSpiller.hasSpilledData() && !hasMoreData()) || !diskSpiller.hasSpilledData());
}
@Override
public long calculateMaxPeekMemory() {
- // In fact, we need to cache all data from input.
- // Now the child of this Operator only will be ShowQueries, it only returns one Block.
return inputOperator.calculateMaxPeekMemory()
- + inputOperator.calculateRetainedSizeAfterCallingNext();
+ + inputOperator.calculateRetainedSizeAfterCallingNext()
+ + sortBufferManager.SORT_BUFFER_SIZE;
}
@Override
public long calculateMaxReturnSize() {
- return inputOperator.calculateMaxReturnSize();
+ return TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
}
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return inputOperator.calculateRetainedSizeAfterCallingNext();
+ return inputOperator.calculateRetainedSizeAfterCallingNext()
+ + sortBufferManager.SORT_BUFFER_SIZE;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186675797
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
Review Comment:
it will be implement in the future.
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
+
+ public FileSpillerReader(String fileName, SortBufferManager sortBufferManager, TsBlockSerde serde)
+ throws FileNotFoundException {
+ this.fileInputStream = new FileInputStream(fileName);
+ this.cacheBlocks = new ArrayList<>();
+ this.rowIndex = 0;
+ this.sortBufferManager = sortBufferManager;
+ this.serde = serde;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = new MergeSortKey(cacheBlocks.get(tsBlockIndex), rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ private boolean readTsBlockFromFile() throws IoTDBException {
+ long bufferSize = sortBufferManager.getReaderBufferAvailable();
+ cacheBlocks.clear();
+ while (bufferSize >= DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES) {
+ long size = read();
+ if (size == -1) break;
+ bufferSize -= size;
+ }
+
+ if (cacheBlocks.isEmpty()) {
+ return false;
+ }
+ rowIndex = 0;
+ tsBlockIndex = 0;
+ return true;
+ }
+
+ private long read() throws IoTDBException {
+ byte[] bytes = new byte[4];
+ try {
+ int readLen = fileInputStream.read(bytes);
+ if (readLen == -1) {
+ return -1;
+ }
+ int capacity = BytesUtils.bytesToInt(bytes);
+ byte[] tsBlockBytes = ReadWriteIOUtils.readBytes(fileInputStream, capacity);
+ ByteBuffer buffer = ByteBuffer.wrap(tsBlockBytes);
+ TsBlock cachedTsBlock = serde.deserialize(buffer);
+ cacheBlocks.add(cachedTsBlock);
+ return cachedTsBlock.getRetainedSizeInBytes();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186673002
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
+ processingTask.add(future);
+ }
+
+ public void spillSortedData(List<MergeSortKey> sortedData) throws IOException {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186669082
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
Review Comment:
it will be solved later
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674219
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
+ }
+ }
+
@Override
public boolean hasNext() throws Exception {
- return inputOperator.hasNextWithTimer() || cachedData != null;
+ return inputOperator.hasNextWithTimer()
+ || (!diskSpiller.hasSpilledData() && curRow != cachedData.size())
+ || (diskSpiller.hasSpilledData() && hasMoreData());
}
@Override
public void close() throws Exception {
+ cachedData = null;
+ clear();
inputOperator.close();
}
@Override
public boolean isFinished() throws Exception {
- return cachedData == null;
+ return inputOperator.isFinished()
+ && (cachedData == null || cachedData.size() == curRow)
+ && ((diskSpiller.hasSpilledData() && !hasMoreData()) || !diskSpiller.hasSpilledData());
}
@Override
public long calculateMaxPeekMemory() {
- // In fact, we need to cache all data from input.
- // Now the child of this Operator only will be ShowQueries, it only returns one Block.
return inputOperator.calculateMaxPeekMemory()
- + inputOperator.calculateRetainedSizeAfterCallingNext();
+ + inputOperator.calculateRetainedSizeAfterCallingNext()
+ + sortBufferManager.SORT_BUFFER_SIZE;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674433
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186675736
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
+
+ public long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
+
+ private long bufferUsed;
+
+ private final long BUFFER_SIZE_FOR_ONE_BRANCH = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+ private final long BUFFER_AVAILABLE_FOR_ALL_BRANCH;
+ private long readerBuffer = 0;
+ private long branchNum = 0;
+
+ public SortBufferManager() {
+ this.BUFFER_AVAILABLE_FOR_ALL_BRANCH = SORT_BUFFER_SIZE - DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ // the initial value is the buffer for output.
+ this.bufferUsed = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ }
+
+ public synchronized void allocateOneSortBranch() {
+ boolean success = allocate(BUFFER_SIZE_FOR_ONE_BRANCH);
+ if (!success) throw new IllegalArgumentException("Not enough memory for sorting");
+ branchNum++;
+ }
+
+ public synchronized boolean check(long size) {
Review Comment:
done
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
+
+ public long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
+
+ private long bufferUsed;
+
+ private final long BUFFER_SIZE_FOR_ONE_BRANCH = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+ private final long BUFFER_AVAILABLE_FOR_ALL_BRANCH;
+ private long readerBuffer = 0;
+ private long branchNum = 0;
+
+ public SortBufferManager() {
+ this.BUFFER_AVAILABLE_FOR_ALL_BRANCH = SORT_BUFFER_SIZE - DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ // the initial value is the buffer for output.
+ this.bufferUsed = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ }
+
+ public synchronized void allocateOneSortBranch() {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186675818
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186676458
##########
server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1057,6 +1057,16 @@ public void loadProperties(Properties properties) {
Boolean.parseBoolean(
properties.getProperty("quota_enable", String.valueOf(conf.isQuotaEnable()))));
+ // the buffer for sort operator to calculate
+ conf.setSortBufferSize(
+ Long.parseLong(
+ properties
+ .getProperty("sort_buffer_size", Long.toString(conf.getSortBufferSize()))
+ .trim()));
+
+ // tmp filePath for sort operator
+ conf.setSortTmpDir(properties.getProperty("sort_tmp_dir", conf.getSortTmpDir()));
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186672928
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674303
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,80 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.db.utils.datastructure.SortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
+
private List<MergeSortKey> cachedData;
- private final Comparator<MergeSortKey> comparator;
+ private final Comparator<SortKey> comparator;
+ private long cachedBytes;
+ private final DiskSpiller diskSpiller;
+ private final SortBufferManager sortBufferManager;
+
+ // For mergeSort
+
+ private MergeSortHeap mergeSortHeap;
+ private List<SortReader> sortReaders;
+ private boolean[] noMoreData;
+ private boolean[] isEmpty;
+
+ Logger logger = LoggerFactory.getLogger(SortOperator.class);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] JackieTien97 commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "JackieTien97 (via GitHub)" <gi...@apache.org>.
JackieTien97 commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1184885483
##########
integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java:
##########
@@ -99,6 +99,7 @@ public class IoTDBOrderByIT {
@BeforeClass
public static void setUp() throws Exception {
+ EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(10 * 1024 * 1024L);
Review Comment:
You'd better add a IoTDBOrderByIT2 extending IoTDBOrderByIT2 whose sortBufferSize is small enough to trigger spilling data to disk.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java:
##########
@@ -404,6 +411,20 @@ private Throwable closeAndDestroyOperators() {
return inFlightException;
}
+ private void cleanTmpFile() throws IOException {
+ String tmpSortDir =
+ IoTDBDescriptor.getInstance().getConfig().getSortTmpDir()
+ + File.separator
+ + driverContext.getFragmentInstanceContext().getId().getFragmentInstanceId()
+ + File.separator;
+ String pipeLineDir = tmpSortDir + driverContext.getPipelineId() + File.separator;
+ Path tmpPipeLineDir = Paths.get(pipeLineDir);
+ if (!Files.exists(tmpPipeLineDir)) return;
+ Files.deleteIfExists(tmpPipeLineDir);
+ Path tmpDir = Paths.get(tmpSortDir);
+ Files.deleteIfExists(tmpDir);
Review Comment:
Do we need delete FI level tmp dir in each Driver?
##########
integration-test/src/test/java/org/apache/iotdb/db/it/orderBy/IoTDBOrderByIT.java:
##########
@@ -99,6 +99,7 @@ public class IoTDBOrderByIT {
@BeforeClass
public static void setUp() throws Exception {
+ EnvFactory.getEnv().getConfig().getDataNodeCommonConfig().setSortBufferSize(1024 * 1024L);
Review Comment:
Add IoTDBOrderByIT2 which extends IoTDBOrderByIT and has a small enough sortBufferSize to trigger spiiling to disk.
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
Review Comment:
use FileChannel instead of FileInputStream and always add private final if necessary.
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
+
+ public FileSpillerReader(String fileName, SortBufferManager sortBufferManager, TsBlockSerde serde)
+ throws FileNotFoundException {
+ this.fileInputStream = new FileInputStream(fileName);
+ this.cacheBlocks = new ArrayList<>();
+ this.rowIndex = 0;
+ this.sortBufferManager = sortBufferManager;
+ this.serde = serde;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = new MergeSortKey(cacheBlocks.get(tsBlockIndex), rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ private boolean readTsBlockFromFile() throws IoTDBException {
+ long bufferSize = sortBufferManager.getReaderBufferAvailable();
+ cacheBlocks.clear();
+ while (bufferSize >= DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES) {
+ long size = read();
+ if (size == -1) break;
+ bufferSize -= size;
+ }
+
+ if (cacheBlocks.isEmpty()) {
+ return false;
+ }
+ rowIndex = 0;
+ tsBlockIndex = 0;
+ return true;
+ }
+
+ private long read() throws IoTDBException {
+ byte[] bytes = new byte[4];
+ try {
+ int readLen = fileInputStream.read(bytes);
+ if (readLen == -1) {
+ return -1;
+ }
+ int capacity = BytesUtils.bytesToInt(bytes);
+ byte[] tsBlockBytes = ReadWriteIOUtils.readBytes(fileInputStream, capacity);
+ ByteBuffer buffer = ByteBuffer.wrap(tsBlockBytes);
+ TsBlock cachedTsBlock = serde.deserialize(buffer);
+ cacheBlocks.add(cachedTsBlock);
+ return cachedTsBlock.getRetainedSizeInBytes();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ @Override
+ public boolean hasNext() throws IoTDBException {
+
+ if (isEnd) return false;
+
+ if (cacheBlocks.isEmpty()
+ || (rowIndex == cacheBlocks.get(tsBlockIndex).getPositionCount()
+ && tsBlockIndex == cacheBlocks.size() - 1)) {
+ boolean hasData = readTsBlockFromFile();
+ if (!hasData) {
+ isEnd = true;
+ return false;
+ }
+ return true;
+ }
+
+ if (rowIndex < cacheBlocks.get(tsBlockIndex).getPositionCount()) {
+ return true;
+ }
+
+ tsBlockIndex++;
+ rowIndex = 0;
+ return true;
+ }
+
+ @Override
+ public void close() throws IoTDBException {
+ try {
+ fileInputStream.close();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
```suggestion
throw new IoTDBException("Failed to close tmp sort file input stream: " + fileName, e, TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
```
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
+
+ public FileSpillerReader(String fileName, SortBufferManager sortBufferManager, TsBlockSerde serde)
+ throws FileNotFoundException {
+ this.fileInputStream = new FileInputStream(fileName);
+ this.cacheBlocks = new ArrayList<>();
+ this.rowIndex = 0;
+ this.sortBufferManager = sortBufferManager;
+ this.serde = serde;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = new MergeSortKey(cacheBlocks.get(tsBlockIndex), rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ private boolean readTsBlockFromFile() throws IoTDBException {
+ long bufferSize = sortBufferManager.getReaderBufferAvailable();
+ cacheBlocks.clear();
+ while (bufferSize >= DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES) {
+ long size = read();
+ if (size == -1) break;
+ bufferSize -= size;
+ }
+
+ if (cacheBlocks.isEmpty()) {
+ return false;
+ }
+ rowIndex = 0;
+ tsBlockIndex = 0;
+ return true;
+ }
+
+ private long read() throws IoTDBException {
+ byte[] bytes = new byte[4];
Review Comment:
use `ByteBuffer.allocate()` instead.
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
Review Comment:
You need to make it clear that whether this class is a global MemoryPool or it's exclusive by one SortOperator.
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
Review Comment:
```suggestion
this.filePrefix = filePrefix + "-";
```
No need to add `this.getClass().getSimpleName()`
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
Actually, we need more details about why failed instead of just printing a meaningless `Failed to spill data to disk`
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
+ processingTask.add(future);
+ }
+
+ public void spillSortedData(List<MergeSortKey> sortedData) throws IOException {
Review Comment:
add a TODO here, it seems that you don't need to copy all the sorted lines into another tsblocks again, you can directly serialize them.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java:
##########
@@ -404,6 +411,20 @@ private Throwable closeAndDestroyOperators() {
return inFlightException;
}
+ private void cleanTmpFile() throws IOException {
Review Comment:
It's not an operation that each query needs, you may need add a field in DriverContext to indicate that whether this driver needs to clean sort tmp dir, because you call at least two IO related system calls in this method which are not very cheap opretions.
##########
server/src/main/java/org/apache/iotdb/db/tools/MemoryReader.java:
##########
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+
+import java.util.List;
+
+public class MemoryReader implements SortReader {
+
+ // all the data in MemoryReader lies in memory
+ private final List<MergeSortKey> cachedData;
+ private int rowIndex;
+
+ public MemoryReader(List<MergeSortKey> cachedTsBlock) {
+ this.cachedData = cachedTsBlock;
+ this.rowIndex = 0;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = cachedData.get(rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ @Override
+ public boolean hasNext() throws IoTDBException {
+ return cachedData != null && rowIndex != cachedData.size();
Review Comment:
add a final field in `MemoryReader` to save `cacheData.size()`, avoid calling `List.size()` every time.
##########
server/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortKey.java:
##########
@@ -21,21 +21,16 @@
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-public class MergeSortKey {
+public class MergeSortKey extends SortKey {
- public TsBlock tsBlock;
- public int rowIndex;
-
- public int columnIndex;
+ public int tsBlockIndex;
Review Comment:
It shouldn't be named as tsBlockIndex because it's also used in SortOperator. It should be a more general name like `mergeChannelIndex`?
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,79 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
Review Comment:
```suggestion
private final String filePrefixPath;
```
no need to save those three characters, readable is more important.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,80 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.db.utils.datastructure.SortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
+
private List<MergeSortKey> cachedData;
Review Comment:
```suggestion
private List<SortKey> cachedData;
```
When it's used in SortReader, we can transform them to MergeSortKey.
##########
server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1057,6 +1057,16 @@ public void loadProperties(Properties properties) {
Boolean.parseBoolean(
properties.getProperty("quota_enable", String.valueOf(conf.isQuotaEnable()))));
+ // the buffer for sort operator to calculate
+ conf.setSortBufferSize(
+ Long.parseLong(
+ properties
+ .getProperty("sort_buffer_size", Long.toString(conf.getSortBufferSize()))
+ .trim()));
+
+ // tmp filePath for sort operator
+ conf.setSortTmpDir(properties.getProperty("sort_tmp_dir", conf.getSortTmpDir()));
Review Comment:
It's a DataNode specific property, you should add it in `server/src/assembly/resources/conf/iotdb-datanode.properties`.
##########
server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1057,6 +1057,16 @@ public void loadProperties(Properties properties) {
Boolean.parseBoolean(
properties.getProperty("quota_enable", String.valueOf(conf.isQuotaEnable()))));
+ // the buffer for sort operator to calculate
+ conf.setSortBufferSize(
+ Long.parseLong(
+ properties
+ .getProperty("sort_buffer_size", Long.toString(conf.getSortBufferSize()))
+ .trim()));
Review Comment:
Think more about where to read this config property from? You need to add this config property in `node-commons/src/assembly/resources/conf/iotdb-common.properties`.
And remember to rename it to `sort_buffer_size_in_mb`.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,80 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.db.utils.datastructure.SortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
+
private List<MergeSortKey> cachedData;
- private final Comparator<MergeSortKey> comparator;
+ private final Comparator<SortKey> comparator;
+ private long cachedBytes;
+ private final DiskSpiller diskSpiller;
+ private final SortBufferManager sortBufferManager;
+
+ // For mergeSort
+
+ private MergeSortHeap mergeSortHeap;
+ private List<SortReader> sortReaders;
+ private boolean[] noMoreData;
+ private boolean[] isEmpty;
+
+ Logger logger = LoggerFactory.getLogger(SortOperator.class);
Review Comment:
```suggestion
private static final Logger logger = LoggerFactory.getLogger(SortOperator.class);
```
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
+
+ public long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
Review Comment:
```suggestion
public static final long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
```
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
Review Comment:
```suggestion
```
Change the method to static, and no need to save this field.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
make Exception more readable and save the root cause.
##########
server/src/main/java/org/apache/iotdb/db/tools/SortReader.java:
##########
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+
+public interface SortReader {
+
+ MergeSortKey next();
+
+ boolean hasNext() throws IoTDBException;
Review Comment:
Add java doc for these two methods, because if caller don't call hasNext before calling next, it will cause fatal error.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
+ }
+ }
+
@Override
public boolean hasNext() throws Exception {
- return inputOperator.hasNextWithTimer() || cachedData != null;
+ return inputOperator.hasNextWithTimer()
+ || (!diskSpiller.hasSpilledData() && curRow != cachedData.size())
+ || (diskSpiller.hasSpilledData() && hasMoreData());
}
@Override
public void close() throws Exception {
+ cachedData = null;
+ clear();
inputOperator.close();
}
@Override
public boolean isFinished() throws Exception {
- return cachedData == null;
+ return inputOperator.isFinished()
+ && (cachedData == null || cachedData.size() == curRow)
+ && ((diskSpiller.hasSpilledData() && !hasMoreData()) || !diskSpiller.hasSpilledData());
}
@Override
public long calculateMaxPeekMemory() {
- // In fact, we need to cache all data from input.
- // Now the child of this Operator only will be ShowQueries, it only returns one Block.
return inputOperator.calculateMaxPeekMemory()
- + inputOperator.calculateRetainedSizeAfterCallingNext();
+ + inputOperator.calculateRetainedSizeAfterCallingNext()
+ + sortBufferManager.SORT_BUFFER_SIZE;
}
@Override
public long calculateMaxReturnSize() {
- return inputOperator.calculateMaxReturnSize();
+ return TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
}
@Override
public long calculateRetainedSizeAfterCallingNext() {
- return inputOperator.calculateRetainedSizeAfterCallingNext();
+ return inputOperator.calculateRetainedSizeAfterCallingNext()
+ + sortBufferManager.SORT_BUFFER_SIZE;
Review Comment:
```suggestion
+ SORT_BUFFER_SIZE;
```
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
+
+ public FileSpillerReader(String fileName, SortBufferManager sortBufferManager, TsBlockSerde serde)
+ throws FileNotFoundException {
+ this.fileInputStream = new FileInputStream(fileName);
+ this.cacheBlocks = new ArrayList<>();
+ this.rowIndex = 0;
+ this.sortBufferManager = sortBufferManager;
+ this.serde = serde;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = new MergeSortKey(cacheBlocks.get(tsBlockIndex), rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ private boolean readTsBlockFromFile() throws IoTDBException {
+ long bufferSize = sortBufferManager.getReaderBufferAvailable();
+ cacheBlocks.clear();
+ while (bufferSize >= DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES) {
+ long size = read();
+ if (size == -1) break;
+ bufferSize -= size;
+ }
+
+ if (cacheBlocks.isEmpty()) {
+ return false;
+ }
+ rowIndex = 0;
+ tsBlockIndex = 0;
+ return true;
+ }
+
+ private long read() throws IoTDBException {
+ byte[] bytes = new byte[4];
+ try {
+ int readLen = fileInputStream.read(bytes);
+ if (readLen == -1) {
+ return -1;
+ }
+ int capacity = BytesUtils.bytesToInt(bytes);
+ byte[] tsBlockBytes = ReadWriteIOUtils.readBytes(fileInputStream, capacity);
+ ByteBuffer buffer = ByteBuffer.wrap(tsBlockBytes);
+ TsBlock cachedTsBlock = serde.deserialize(buffer);
+ cacheBlocks.add(cachedTsBlock);
+ return cachedTsBlock.getRetainedSizeInBytes();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
more detail error messages, same as below.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,80 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.db.utils.datastructure.SortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
+
private List<MergeSortKey> cachedData;
- private final Comparator<MergeSortKey> comparator;
+ private final Comparator<SortKey> comparator;
+ private long cachedBytes;
+ private final DiskSpiller diskSpiller;
+ private final SortBufferManager sortBufferManager;
+
+ // For mergeSort
+
+ private MergeSortHeap mergeSortHeap;
+ private List<SortReader> sortReaders;
+ private boolean[] noMoreData;
+ private boolean[] isEmpty;
Review Comment:
```suggestion
```
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
Review Comment:
This file won't exist ever. It's a file prefix path, not a directory name.
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
+
+ public long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
+
+ private long bufferUsed;
+
+ private final long BUFFER_SIZE_FOR_ONE_BRANCH = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+ private final long BUFFER_AVAILABLE_FOR_ALL_BRANCH;
+ private long readerBuffer = 0;
+ private long branchNum = 0;
+
+ public SortBufferManager() {
+ this.BUFFER_AVAILABLE_FOR_ALL_BRANCH = SORT_BUFFER_SIZE - DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ // the initial value is the buffer for output.
+ this.bufferUsed = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ }
+
+ public synchronized void allocateOneSortBranch() {
Review Comment:
Do we really need this synchronized, because I find that now you new one SortBufferManager for each SortOperator.
##########
server/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortKey.java:
##########
@@ -21,21 +21,16 @@
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-public class MergeSortKey {
+public class MergeSortKey extends SortKey {
- public TsBlock tsBlock;
- public int rowIndex;
-
- public int columnIndex;
+ public int tsBlockIndex;
Review Comment:
BTW, add some comments about this field to make it more readable.
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
+
+ public long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
+
+ private long bufferUsed;
+
+ private final long BUFFER_SIZE_FOR_ONE_BRANCH = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+ private final long BUFFER_AVAILABLE_FOR_ALL_BRANCH;
+ private long readerBuffer = 0;
+ private long branchNum = 0;
+
+ public SortBufferManager() {
+ this.BUFFER_AVAILABLE_FOR_ALL_BRANCH = SORT_BUFFER_SIZE - DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ // the initial value is the buffer for output.
+ this.bufferUsed = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ }
+
+ public synchronized void allocateOneSortBranch() {
Review Comment:
do this check for all the following methods.
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
Review Comment:
The reason to use `FileChannel` is that later we may use BufferPool to allocate DirectByteBuffer and only `FileChannel` support reading into ByteBuffer.
About how to use `FileChannel`, you can refer to `LocalTsFileInput`.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
Review Comment:
Actually, this code block is used to init MergeSortHeap, it can be put into
```
if (mergeSortHeap == null) {
}
```
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
+
+ public FileSpillerReader(String fileName, SortBufferManager sortBufferManager, TsBlockSerde serde)
+ throws FileNotFoundException {
+ this.fileInputStream = new FileInputStream(fileName);
+ this.cacheBlocks = new ArrayList<>();
+ this.rowIndex = 0;
+ this.sortBufferManager = sortBufferManager;
+ this.serde = serde;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = new MergeSortKey(cacheBlocks.get(tsBlockIndex), rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ private boolean readTsBlockFromFile() throws IoTDBException {
+ long bufferSize = sortBufferManager.getReaderBufferAvailable();
+ cacheBlocks.clear();
+ while (bufferSize >= DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES) {
+ long size = read();
+ if (size == -1) break;
+ bufferSize -= size;
+ }
+
+ if (cacheBlocks.isEmpty()) {
+ return false;
+ }
+ rowIndex = 0;
+ tsBlockIndex = 0;
+ return true;
+ }
+
+ private long read() throws IoTDBException {
+ byte[] bytes = new byte[4];
+ try {
+ int readLen = fileInputStream.read(bytes);
+ if (readLen == -1) {
+ return -1;
+ }
+ int capacity = BytesUtils.bytesToInt(bytes);
+ byte[] tsBlockBytes = ReadWriteIOUtils.readBytes(fileInputStream, capacity);
+ ByteBuffer buffer = ByteBuffer.wrap(tsBlockBytes);
+ TsBlock cachedTsBlock = serde.deserialize(buffer);
+ cacheBlocks.add(cachedTsBlock);
+ return cachedTsBlock.getRetainedSizeInBytes();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ @Override
+ public boolean hasNext() throws IoTDBException {
+
+ if (isEnd) return false;
+
+ if (cacheBlocks.isEmpty()
+ || (rowIndex == cacheBlocks.get(tsBlockIndex).getPositionCount()
+ && tsBlockIndex == cacheBlocks.size() - 1)) {
+ boolean hasData = readTsBlockFromFile();
+ if (!hasData) {
+ isEnd = true;
+ return false;
+ }
+ return true;
+ }
+
+ if (rowIndex < cacheBlocks.get(tsBlockIndex).getPositionCount()) {
+ return true;
+ }
+
+ tsBlockIndex++;
+ rowIndex = 0;
+ return true;
+ }
+
+ @Override
+ public void close() throws IoTDBException {
+ try {
+ fileInputStream.close();
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
Always remember that we should make error message more readable, and if possible, we should save the root cause instead of ignore it.
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
+
+ public long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
+
+ private long bufferUsed;
+
+ private final long BUFFER_SIZE_FOR_ONE_BRANCH = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+ private final long BUFFER_AVAILABLE_FOR_ALL_BRANCH;
+ private long readerBuffer = 0;
+ private long branchNum = 0;
+
+ public SortBufferManager() {
+ this.BUFFER_AVAILABLE_FOR_ALL_BRANCH = SORT_BUFFER_SIZE - DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ // the initial value is the buffer for output.
+ this.bufferUsed = DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+ }
+
+ public synchronized void allocateOneSortBranch() {
+ boolean success = allocate(BUFFER_SIZE_FOR_ONE_BRANCH);
+ if (!success) throw new IllegalArgumentException("Not enough memory for sorting");
+ branchNum++;
+ }
+
+ public synchronized boolean check(long size) {
Review Comment:
```suggestion
private boolean check(long size) {
```
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
Review Comment:
```suggestion
private void createFolder(String folderPath) throws IOException {
```
check for yourself if you really need some methods to be public.
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
Review Comment:
no need to save sortBufferManager here, you only use `ReaderBufferAvailable` in SortBufferManager.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,80 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.db.utils.datastructure.SortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
+
private List<MergeSortKey> cachedData;
- private final Comparator<MergeSortKey> comparator;
+ private final Comparator<SortKey> comparator;
+ private long cachedBytes;
+ private final DiskSpiller diskSpiller;
+ private final SortBufferManager sortBufferManager;
+
+ // For mergeSort
+
+ private MergeSortHeap mergeSortHeap;
+ private List<SortReader> sortReaders;
+ private boolean[] noMoreData;
+ private boolean[] isEmpty;
Review Comment:
Think more about it, do we really need these two?
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
Review Comment:
And `if (noMoreData[i] || !isEmpty[i]) continue;` can be deleted.
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
Review Comment:
```suggestion
private void spill(List<TsBlock> tsBlocks) throws IOException {
```
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
Review Comment:
Why use async style and actually directExecutor() will using current working thread to do this.
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
+ processingTask.add(future);
+ }
+
+ public void spillSortedData(List<MergeSortKey> sortedData) throws IOException {
+ List<TsBlock> tsBlocks = new ArrayList<>();
+ TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(dataTypeList);
+ ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ ColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+
+ for (MergeSortKey mergeSortKey : sortedData) {
+ writeMergeSortKey(mergeSortKey, columnBuilders, timeColumnBuilder);
+ tsBlockBuilder.declarePosition();
+ if (tsBlockBuilder.isFull()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ tsBlockBuilder.reset();
+ timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ }
+ }
+
+ if (!tsBlockBuilder.isEmpty()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ }
+
+ spill(tsBlocks);
+ }
+
+ private boolean writeData(List<TsBlock> sortedData, String fileName) {
+ try {
+ Path filePath = Paths.get(fileName);
+ Files.createFile(filePath);
+
+ try (FileOutputStream fileOutputStream = new FileOutputStream(fileName)) {
+ for (TsBlock tsBlock : sortedData) {
+ ByteBuffer tsBlockBuffer = serde.serialize(tsBlock);
+ ReadWriteIOUtils.write(tsBlockBuffer, fileOutputStream);
+ }
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ }
+
+ private void writeMergeSortKey(
+ MergeSortKey mergeSortKey, ColumnBuilder[] columnBuilders, ColumnBuilder timeColumnBuilder) {
+ timeColumnBuilder.writeLong(mergeSortKey.tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < columnBuilders.length; i++) {
+ if (mergeSortKey.tsBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ columnBuilders[i].appendNull();
+ } else {
+ columnBuilders[i].write(mergeSortKey.tsBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ }
+ }
+
+ public boolean hasSpilledData() {
+ return !fileIndex.isEmpty();
+ }
+
+ public List<String> getFilePaths() {
Review Comment:
```suggestion
private List<String> getFilePaths() {
```
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
Review Comment:
same as below.
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
+ }
+ }
+
@Override
public boolean hasNext() throws Exception {
- return inputOperator.hasNextWithTimer() || cachedData != null;
+ return inputOperator.hasNextWithTimer()
+ || (!diskSpiller.hasSpilledData() && curRow != cachedData.size())
+ || (diskSpiller.hasSpilledData() && hasMoreData());
}
@Override
public void close() throws Exception {
+ cachedData = null;
+ clear();
inputOperator.close();
}
@Override
public boolean isFinished() throws Exception {
- return cachedData == null;
+ return inputOperator.isFinished()
+ && (cachedData == null || cachedData.size() == curRow)
+ && ((diskSpiller.hasSpilledData() && !hasMoreData()) || !diskSpiller.hasSpilledData());
}
@Override
public long calculateMaxPeekMemory() {
- // In fact, we need to cache all data from input.
- // Now the child of this Operator only will be ShowQueries, it only returns one Block.
return inputOperator.calculateMaxPeekMemory()
- + inputOperator.calculateRetainedSizeAfterCallingNext();
+ + inputOperator.calculateRetainedSizeAfterCallingNext()
+ + sortBufferManager.SORT_BUFFER_SIZE;
Review Comment:
```suggestion
+SORT_BUFFER_SIZE;
```
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
Review Comment:
```suggestion
```
No need to save it, its contents must be lile `1, 2, 3,...index`
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
+ processingTask.add(future);
+ }
+
+ public void spillSortedData(List<MergeSortKey> sortedData) throws IOException {
+ List<TsBlock> tsBlocks = new ArrayList<>();
+ TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(dataTypeList);
+ ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ ColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+
+ for (MergeSortKey mergeSortKey : sortedData) {
+ writeMergeSortKey(mergeSortKey, columnBuilders, timeColumnBuilder);
+ tsBlockBuilder.declarePosition();
+ if (tsBlockBuilder.isFull()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ tsBlockBuilder.reset();
+ timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ }
+ }
+
+ if (!tsBlockBuilder.isEmpty()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ }
+
+ spill(tsBlocks);
+ }
+
+ private boolean writeData(List<TsBlock> sortedData, String fileName) {
+ try {
+ Path filePath = Paths.get(fileName);
+ Files.createFile(filePath);
+
+ try (FileOutputStream fileOutputStream = new FileOutputStream(fileName)) {
+ for (TsBlock tsBlock : sortedData) {
+ ByteBuffer tsBlockBuffer = serde.serialize(tsBlock);
+ ReadWriteIOUtils.write(tsBlockBuffer, fileOutputStream);
+ }
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ }
+
+ private void writeMergeSortKey(
+ MergeSortKey mergeSortKey, ColumnBuilder[] columnBuilders, ColumnBuilder timeColumnBuilder) {
+ timeColumnBuilder.writeLong(mergeSortKey.tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < columnBuilders.length; i++) {
+ if (mergeSortKey.tsBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ columnBuilders[i].appendNull();
+ } else {
+ columnBuilders[i].write(mergeSortKey.tsBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ }
+ }
+
+ public boolean hasSpilledData() {
+ return !fileIndex.isEmpty();
+ }
+
+ public List<String> getFilePaths() {
+ List<String> filePaths = new ArrayList<>();
+ for (int index : fileIndex) {
+ filePaths.add(filePrefix + index + fileSuffix);
+ }
+ return filePaths;
+ }
+
+ public List<SortReader> getReaders(SortBufferManager sortBufferManager)
+ throws FileNotFoundException {
+ List<String> filePaths = getFilePaths();
+ List<SortReader> sortReaders = new ArrayList<>();
+ for (String filePath : filePaths) {
+ sortReaders.add(new FileSpillerReader(filePath, sortBufferManager, serde));
+ }
+ return sortReaders;
+ }
+
+ public void clear() throws IOException {
+ List<String> filePaths = getFilePaths();
+ for (String filePath : filePaths) {
+ Path newPath = Paths.get(filePath);
+ Files.deleteIfExists(newPath);
+ }
+ }
Review Comment:
No need to delete them one by one, we only need to delete one time in SortOperator(delete the corresponding folder).
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
Review Comment:
```suggestion
logger.error("Failed to delete spilled file", e);
```
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
Review Comment:
```suggestion
```
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
```suggestion
throw new IoTDBException("Failed to spill intermediate sorted result", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode(), e);
```
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
+ if (sortReaders != null) {
+ for (SortReader sortReader : sortReaders) {
+ sortReader.close();
+ }
+ }
+ } catch (Exception e) {
+ logger.error("delete spilled file error: {}", e.getMessage());
+ }
+ }
+
@Override
public boolean hasNext() throws Exception {
- return inputOperator.hasNextWithTimer() || cachedData != null;
+ return inputOperator.hasNextWithTimer()
+ || (!diskSpiller.hasSpilledData() && curRow != cachedData.size())
+ || (diskSpiller.hasSpilledData() && hasMoreData());
}
@Override
public void close() throws Exception {
+ cachedData = null;
+ clear();
inputOperator.close();
}
@Override
public boolean isFinished() throws Exception {
- return cachedData == null;
+ return inputOperator.isFinished()
+ && (cachedData == null || cachedData.size() == curRow)
+ && ((diskSpiller.hasSpilledData() && !hasMoreData()) || !diskSpiller.hasSpilledData());
Review Comment:
```suggestion
&& (!diskSpiller.hasSpilledData() || !hasMoreData());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186666679
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java:
##########
@@ -404,6 +411,20 @@ private Throwable closeAndDestroyOperators() {
return inFlightException;
}
+ private void cleanTmpFile() throws IOException {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186669082
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
Review Comment:
it will be solved in the future.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674799
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186667486
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/driver/Driver.java:
##########
@@ -404,6 +411,20 @@ private Throwable closeAndDestroyOperators() {
return inFlightException;
}
+ private void cleanTmpFile() throws IOException {
+ String tmpSortDir =
+ IoTDBDescriptor.getInstance().getConfig().getSortTmpDir()
+ + File.separator
+ + driverContext.getFragmentInstanceContext().getId().getFragmentInstanceId()
+ + File.separator;
+ String pipeLineDir = tmpSortDir + driverContext.getPipelineId() + File.separator;
+ Path tmpPipeLineDir = Paths.get(pipeLineDir);
+ if (!Files.exists(tmpPipeLineDir)) return;
+ Files.deleteIfExists(tmpPipeLineDir);
+ Path tmpDir = Paths.get(tmpSortDir);
+ Files.deleteIfExists(tmpDir);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186668582
##########
server/src/main/java/org/apache/iotdb/db/tools/SortBufferManager.java:
##########
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class SortBufferManager {
+
+ public long SORT_BUFFER_SIZE = IoTDBDescriptor.getInstance().getConfig().getSortBufferSize();
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186670343
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674752
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
done
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186675358
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,80 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.db.utils.datastructure.SortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
+
private List<MergeSortKey> cachedData;
- private final Comparator<MergeSortKey> comparator;
+ private final Comparator<SortKey> comparator;
+ private long cachedBytes;
+ private final DiskSpiller diskSpiller;
+ private final SortBufferManager sortBufferManager;
+
+ // For mergeSort
+
+ private MergeSortHeap mergeSortHeap;
+ private List<SortReader> sortReaders;
+ private boolean[] noMoreData;
+ private boolean[] isEmpty;
Review Comment:
isEmpty is useless, and noMoreData is retained to be used in hasMoreData() in hasNext().
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186585658
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,79 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186668345
##########
server/src/main/java/org/apache/iotdb/db/utils/datastructure/MergeSortKey.java:
##########
@@ -21,21 +21,16 @@
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
-public class MergeSortKey {
+public class MergeSortKey extends SortKey {
- public TsBlock tsBlock;
- public int rowIndex;
-
- public int columnIndex;
+ public int tsBlockIndex;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186673112
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
+ processingTask.add(future);
+ }
+
+ public void spillSortedData(List<MergeSortKey> sortedData) throws IOException {
+ List<TsBlock> tsBlocks = new ArrayList<>();
+ TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(dataTypeList);
+ ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ ColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+
+ for (MergeSortKey mergeSortKey : sortedData) {
+ writeMergeSortKey(mergeSortKey, columnBuilders, timeColumnBuilder);
+ tsBlockBuilder.declarePosition();
+ if (tsBlockBuilder.isFull()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ tsBlockBuilder.reset();
+ timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ }
+ }
+
+ if (!tsBlockBuilder.isEmpty()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ }
+
+ spill(tsBlocks);
+ }
+
+ private boolean writeData(List<TsBlock> sortedData, String fileName) {
+ try {
+ Path filePath = Paths.get(fileName);
+ Files.createFile(filePath);
+
+ try (FileOutputStream fileOutputStream = new FileOutputStream(fileName)) {
+ for (TsBlock tsBlock : sortedData) {
+ ByteBuffer tsBlockBuffer = serde.serialize(tsBlock);
+ ReadWriteIOUtils.write(tsBlockBuffer, fileOutputStream);
+ }
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ }
+
+ private void writeMergeSortKey(
+ MergeSortKey mergeSortKey, ColumnBuilder[] columnBuilders, ColumnBuilder timeColumnBuilder) {
+ timeColumnBuilder.writeLong(mergeSortKey.tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < columnBuilders.length; i++) {
+ if (mergeSortKey.tsBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ columnBuilders[i].appendNull();
+ } else {
+ columnBuilders[i].write(mergeSortKey.tsBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ }
+ }
+
+ public boolean hasSpilledData() {
+ return !fileIndex.isEmpty();
+ }
+
+ public List<String> getFilePaths() {
+ List<String> filePaths = new ArrayList<>();
+ for (int index : fileIndex) {
+ filePaths.add(filePrefix + index + fileSuffix);
+ }
+ return filePaths;
+ }
+
+ public List<SortReader> getReaders(SortBufferManager sortBufferManager)
+ throws FileNotFoundException {
+ List<String> filePaths = getFilePaths();
+ List<SortReader> sortReaders = new ArrayList<>();
+ for (String filePath : filePaths) {
+ sortReaders.add(new FileSpillerReader(filePath, sortBufferManager, serde));
+ }
+ return sortReaders;
+ }
+
+ public void clear() throws IOException {
+ List<String> filePaths = getFilePaths();
+ for (String filePath : filePaths) {
+ Path newPath = Paths.get(filePath);
+ Files.deleteIfExists(newPath);
+ }
+ }
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186672915
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
Review Comment:
done
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] JackieTien97 merged pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "JackieTien97 (via GitHub)" <gi...@apache.org>.
JackieTien97 merged PR #9746:
URL: https://github.com/apache/iotdb/pull/9746
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186673081
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
+ private final List<TSDataType> dataTypeList;
+ private final List<ListenableFuture<Boolean>> processingTask;
+ private final String folderPath;
+ private final String filePrefix;
+ private final String fileSuffix = ".sortTemp";
+
+ private int index;
+ private boolean folderCreated = false;
+ private boolean allProcessingTaskFinished = false;
+ private final TsBlockSerde serde = new TsBlockSerde();
+
+ public DiskSpiller(String folderPath, String filePrefix, List<TSDataType> dataTypeList) {
+ this.folderPath = folderPath;
+ this.filePrefix = filePrefix + this.getClass().getSimpleName() + "-";
+ this.index = 0;
+ this.dataTypeList = dataTypeList;
+ this.fileIndex = new ArrayList<>();
+ this.processingTask = new ArrayList<>();
+ }
+
+ public void createFolder(String folderPath) throws IOException {
+ Path path = Paths.get(folderPath);
+ Files.createDirectories(path);
+ folderCreated = true;
+ }
+
+ public boolean allProcessingTaskFinished() throws IoTDBException {
+ if (allProcessingTaskFinished) return true;
+ for (Future<Boolean> future : processingTask) {
+ if (!future.isDone()) return false;
+ // check if there is exception in the processing task
+ try {
+ boolean finished = future.get();
+ if (!finished) {
+ throw new IoTDBException(
+ "Failed to spill data to disk", TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ } catch (Exception e) {
+ throw new IoTDBException(
+ e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+ allProcessingTaskFinished = true;
+ return true;
+ }
+
+ public synchronized void spill(List<TsBlock> tsBlocks) throws IOException {
+ if (!folderCreated) {
+ createFolder(folderPath);
+ }
+ String fileName = filePrefix + index + fileSuffix;
+ fileIndex.add(index);
+ index++;
+
+ ListenableFuture<Boolean> future =
+ Futures.submit(() -> writeData(tsBlocks, fileName), directExecutor());
+ processingTask.add(future);
+ }
+
+ public void spillSortedData(List<MergeSortKey> sortedData) throws IOException {
+ List<TsBlock> tsBlocks = new ArrayList<>();
+ TsBlockBuilder tsBlockBuilder = new TsBlockBuilder(dataTypeList);
+ ColumnBuilder[] columnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ ColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+
+ for (MergeSortKey mergeSortKey : sortedData) {
+ writeMergeSortKey(mergeSortKey, columnBuilders, timeColumnBuilder);
+ tsBlockBuilder.declarePosition();
+ if (tsBlockBuilder.isFull()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ tsBlockBuilder.reset();
+ timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ }
+ }
+
+ if (!tsBlockBuilder.isEmpty()) {
+ tsBlocks.add(tsBlockBuilder.build());
+ }
+
+ spill(tsBlocks);
+ }
+
+ private boolean writeData(List<TsBlock> sortedData, String fileName) {
+ try {
+ Path filePath = Paths.get(fileName);
+ Files.createFile(filePath);
+
+ try (FileOutputStream fileOutputStream = new FileOutputStream(fileName)) {
+ for (TsBlock tsBlock : sortedData) {
+ ByteBuffer tsBlockBuffer = serde.serialize(tsBlock);
+ ReadWriteIOUtils.write(tsBlockBuffer, fileOutputStream);
+ }
+ }
+ } catch (Exception e) {
+ return false;
+ }
+ return true;
+ }
+
+ private void writeMergeSortKey(
+ MergeSortKey mergeSortKey, ColumnBuilder[] columnBuilders, ColumnBuilder timeColumnBuilder) {
+ timeColumnBuilder.writeLong(mergeSortKey.tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < columnBuilders.length; i++) {
+ if (mergeSortKey.tsBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ columnBuilders[i].appendNull();
+ } else {
+ columnBuilders[i].write(mergeSortKey.tsBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ }
+ }
+
+ public boolean hasSpilledData() {
+ return !fileIndex.isEmpty();
+ }
+
+ public List<String> getFilePaths() {
Review Comment:
done
##########
server/src/main/java/org/apache/iotdb/db/tools/DiskSpiller.java:
##########
@@ -0,0 +1,191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Future;
+
+import static org.weakref.jmx.internal.guava.util.concurrent.MoreExecutors.directExecutor;
+
+public class DiskSpiller {
+
+ private final List<Integer> fileIndex;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674467
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -67,75 +108,253 @@ public ListenableFuture<?> isBlocked() {
public TsBlock next() throws Exception {
if (!inputOperator.hasNextWithTimer()) {
- if (cachedData.size() > 1) {
- cachedData.sort(comparator);
+ if (diskSpiller.hasSpilledData()) {
+ try {
+ prepareSortReaders();
+ return mergeSort();
+ } catch (Exception e) {
+ clear();
+ throw e;
+ }
+ } else {
+ if (curRow == -1) {
+ cachedData.sort(comparator);
+ curRow = 0;
+ }
+ return buildTsBlockInMemory();
}
- TsBlock result = buildTsBlock();
- cachedData = null;
- return result;
}
TsBlock tsBlock = inputOperator.nextWithTimer();
if (tsBlock == null) {
return null;
}
- // add data of each TsBlock from child into list
- for (int i = 0; i < tsBlock.getPositionCount(); i++) {
- cachedData.add(new MergeSortKey(tsBlock, i));
+
+ try {
+ cacheTsBlock(tsBlock);
+ } catch (IoTDBException e) {
+ clear();
+ throw e;
}
+
return null;
}
- private TsBlock buildTsBlock() {
+ private void prepareSortReaders() throws IoTDBException {
+ if (sortReaders != null) return;
+
+ try {
+
+ sortReaders = new ArrayList<>();
+ if (cachedBytes != 0) {
+ cachedData.sort(comparator);
+ if (sortBufferManager.allocate(cachedBytes)) {
+ sortReaders.add(new MemoryReader(cachedData));
+ } else {
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ cachedData = null;
+ }
+ }
+ sortReaders.addAll(diskSpiller.getReaders(sortBufferManager));
+ // if reader is finished
+ noMoreData = new boolean[sortReaders.size()];
+ // need to read data from reader when isEmpty is true
+ isEmpty = new boolean[sortReaders.size()];
+ Arrays.fill(isEmpty, true);
+ } catch (Exception e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private void cacheTsBlock(TsBlock tsBlock) throws IoTDBException {
+ long bytesSize = tsBlock.getRetainedSizeInBytes();
+ if (bytesSize + cachedBytes < sortBufferManager.SORT_BUFFER_SIZE) {
+ cachedBytes += bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ } else {
+ cachedData.sort(comparator);
+ spill();
+ cachedData.clear();
+ cachedBytes = bytesSize;
+ for (int i = 0; i < tsBlock.getPositionCount(); i++) {
+ cachedData.add(new MergeSortKey(tsBlock, i));
+ }
+ }
+ }
+
+ private void spill() throws IoTDBException {
+ try {
+ // if current memory cannot put this tsBlock, an exception will be thrown in spillSortedData()
+ // because there should be at least tsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES for
+ // one branch.
+ sortBufferManager.allocateOneSortBranch();
+ diskSpiller.spillSortedData(cachedData);
+ } catch (IOException e) {
+ throw new IoTDBException(e.getMessage(), TSStatusCode.INTERNAL_SERVER_ERROR.getStatusCode());
+ }
+ }
+
+ private TsBlock buildTsBlockInMemory() {
+ tsBlockBuilder.reset();
TimeColumnBuilder timeColumnBuilder = tsBlockBuilder.getTimeColumnBuilder();
ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
- cachedData.forEach(
- mergeSortKey -> {
- TsBlock tsBlock = mergeSortKey.tsBlock;
- int row = mergeSortKey.rowIndex;
- timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(row));
- for (int i = 0; i < valueColumnBuilders.length; i++) {
- if (tsBlock.getColumn(i).isNull(row)) {
- valueColumnBuilders[i].appendNull();
- continue;
- }
- valueColumnBuilders[i].write(tsBlock.getColumn(i), row);
- }
- tsBlockBuilder.declarePosition();
- });
+ for (int i = curRow; i < cachedData.size(); i++) {
+ MergeSortKey mergeSortKey = cachedData.get(i);
+ TsBlock tsBlock = mergeSortKey.tsBlock;
+ timeColumnBuilder.writeLong(tsBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int j = 0; j < valueColumnBuilders.length; j++) {
+ if (tsBlock.getColumn(j).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[j].appendNull();
+ continue;
+ }
+ valueColumnBuilders[j].write(tsBlock.getColumn(j), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+ curRow++;
+ if (tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
+
+ return tsBlockBuilder.build();
+ }
+
+ private TsBlock mergeSort() throws IoTDBException {
+
+ if (!diskSpiller.allProcessingTaskFinished()) return null;
+
+ if (mergeSortHeap == null) {
+ mergeSortHeap = new MergeSortHeap(sortReaders.size(), comparator);
+ }
+
+ long startTime = System.nanoTime();
+ long maxRuntime = operatorContext.getMaxRunTime().roundTo(TimeUnit.NANOSECONDS);
+
+ // 1. fill the input from each reader
+ for (int i = 0; i < sortReaders.size(); i++) {
+ if (noMoreData[i] || !isEmpty[i]) continue;
+ SortReader sortReader = sortReaders.get(i);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = i;
+ mergeSortHeap.push(mergeSortKey);
+ isEmpty[i] = false;
+ } else {
+ noMoreData[i] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+ }
+
+ // 2. do merge sort until one TsBlock is consumed up
+ tsBlockBuilder.reset();
+ TimeColumnBuilder timeBuilder = tsBlockBuilder.getTimeColumnBuilder();
+ ColumnBuilder[] valueColumnBuilders = tsBlockBuilder.getValueColumnBuilders();
+ while (!mergeSortHeap.isEmpty()) {
+
+ MergeSortKey mergeSortKey = mergeSortHeap.poll();
+ TsBlock targetBlock = mergeSortKey.tsBlock;
+ timeBuilder.writeLong(targetBlock.getTimeByIndex(mergeSortKey.rowIndex));
+ for (int i = 0; i < valueColumnBuilders.length; i++) {
+ if (targetBlock.getColumn(i).isNull(mergeSortKey.rowIndex)) {
+ valueColumnBuilders[i].appendNull();
+ continue;
+ }
+ valueColumnBuilders[i].write(targetBlock.getColumn(i), mergeSortKey.rowIndex);
+ }
+ tsBlockBuilder.declarePosition();
+
+ int readerIndex = mergeSortKey.tsBlockIndex;
+ mergeSortKey = readNextMergeSortKey(readerIndex);
+ if (mergeSortKey != null) {
+ mergeSortHeap.push(mergeSortKey);
+ } else {
+ noMoreData[readerIndex] = true;
+ sortBufferManager.releaseOneSortBranch();
+ }
+
+ // break if time is out or tsBlockBuilder is full or sortBuffer is not enough
+ if (System.nanoTime() - startTime > maxRuntime || tsBlockBuilder.isFull()) {
+ break;
+ }
+ }
return tsBlockBuilder.build();
}
+ private MergeSortKey readNextMergeSortKey(int readerIndex) throws IoTDBException {
+ SortReader sortReader = sortReaders.get(readerIndex);
+ if (sortReader.hasNext()) {
+ MergeSortKey mergeSortKey = sortReader.next();
+ mergeSortKey.tsBlockIndex = readerIndex;
+ return mergeSortKey;
+ }
+ return null;
+ }
+
+ private boolean hasMoreData() {
+ if (noMoreData == null) return true;
+ for (boolean noMore : noMoreData) {
+ if (!noMore) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public void clear() {
+ if (!diskSpiller.hasSpilledData()) return;
+ try {
+ diskSpiller.clear();
+ Path path = Paths.get(filePrePath);
+ Files.deleteIfExists(path);
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186674106
##########
server/src/main/java/org/apache/iotdb/db/mpp/execution/operator/process/SortOperator.java:
##########
@@ -18,39 +18,80 @@
*/
package org.apache.iotdb.db.mpp.execution.operator.process;
+import org.apache.iotdb.commons.exception.IoTDBException;
import org.apache.iotdb.db.mpp.execution.operator.Operator;
import org.apache.iotdb.db.mpp.execution.operator.OperatorContext;
+import org.apache.iotdb.db.tools.DiskSpiller;
+import org.apache.iotdb.db.tools.MemoryReader;
+import org.apache.iotdb.db.tools.SortBufferManager;
+import org.apache.iotdb.db.tools.SortReader;
+import org.apache.iotdb.db.utils.datastructure.MergeSortHeap;
import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.db.utils.datastructure.SortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.block.TsBlock;
import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilder;
+import org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus;
import org.apache.iotdb.tsfile.read.common.block.column.ColumnBuilder;
import org.apache.iotdb.tsfile.read.common.block.column.TimeColumnBuilder;
import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Comparator;
import java.util.List;
+import java.util.concurrent.TimeUnit;
public class SortOperator implements ProcessOperator {
private final OperatorContext operatorContext;
private final Operator inputOperator;
private final TsBlockBuilder tsBlockBuilder;
+ // Use to output the result in memory.
+ // Because the memory may be larger than tsBlockBuilder's max size
+ // so the data may be return in multiple times.
+ private int curRow = -1;
+
+ private final String filePrePath;
+
private List<MergeSortKey> cachedData;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186675102
##########
server/src/main/java/org/apache/iotdb/db/tools/SortReader.java:
##########
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+
+public interface SortReader {
+
+ MergeSortKey next();
+
+ boolean hasNext() throws IoTDBException;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186670301
##########
server/src/main/java/org/apache/iotdb/db/tools/FileSpillerReader.java:
##########
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.commons.exception.IoTDBException;
+import org.apache.iotdb.db.utils.datastructure.MergeSortKey;
+import org.apache.iotdb.rpc.TSStatusCode;
+import org.apache.iotdb.tsfile.read.common.block.TsBlock;
+import org.apache.iotdb.tsfile.read.common.block.column.TsBlockSerde;
+import org.apache.iotdb.tsfile.utils.BytesUtils;
+import org.apache.iotdb.tsfile.utils.ReadWriteIOUtils;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.iotdb.tsfile.read.common.block.TsBlockBuilderStatus.DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES;
+
+public class FileSpillerReader implements SortReader {
+
+ FileInputStream fileInputStream;
+ private final List<TsBlock> cacheBlocks;
+ private int tsBlockIndex;
+ private int rowIndex;
+ private boolean isEnd = false;
+ private final SortBufferManager sortBufferManager;
+ private final TsBlockSerde serde;
+
+ public FileSpillerReader(String fileName, SortBufferManager sortBufferManager, TsBlockSerde serde)
+ throws FileNotFoundException {
+ this.fileInputStream = new FileInputStream(fileName);
+ this.cacheBlocks = new ArrayList<>();
+ this.rowIndex = 0;
+ this.sortBufferManager = sortBufferManager;
+ this.serde = serde;
+ }
+
+ @Override
+ public MergeSortKey next() {
+ MergeSortKey sortKey = new MergeSortKey(cacheBlocks.get(tsBlockIndex), rowIndex);
+ rowIndex++;
+ return sortKey;
+ }
+
+ private boolean readTsBlockFromFile() throws IoTDBException {
+ long bufferSize = sortBufferManager.getReaderBufferAvailable();
+ cacheBlocks.clear();
+ while (bufferSize >= DEFAULT_MAX_TSBLOCK_SIZE_IN_BYTES) {
+ long size = read();
+ if (size == -1) break;
+ bufferSize -= size;
+ }
+
+ if (cacheBlocks.isEmpty()) {
+ return false;
+ }
+ rowIndex = 0;
+ tsBlockIndex = 0;
+ return true;
+ }
+
+ private long read() throws IoTDBException {
+ byte[] bytes = new byte[4];
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [iotdb] ycycse commented on a diff in pull request #9746: [IOTDB-5765] Support Order By Expression [BE Part]
Posted by "ycycse (via GitHub)" <gi...@apache.org>.
ycycse commented on code in PR #9746:
URL: https://github.com/apache/iotdb/pull/9746#discussion_r1186677486
##########
server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java:
##########
@@ -1057,6 +1057,16 @@ public void loadProperties(Properties properties) {
Boolean.parseBoolean(
properties.getProperty("quota_enable", String.valueOf(conf.isQuotaEnable()))));
+ // the buffer for sort operator to calculate
+ conf.setSortBufferSize(
+ Long.parseLong(
+ properties
+ .getProperty("sort_buffer_size", Long.toString(conf.getSortBufferSize()))
+ .trim()));
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@iotdb.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org