You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2020/01/06 13:48:29 UTC
[incubator-iotdb] branch fix_401_correct_chunk_size created (now
99f9ef9)
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a change to branch fix_401_correct_chunk_size
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at 99f9ef9 Merge branch 'master' of github.com:apache/incubator-iotdb
This branch includes the following new commits:
new 6d65b74 IOTDB-401, skip the size of a chunk header if the chunk has no data points
new 99f9ef9 Merge branch 'master' of github.com:apache/incubator-iotdb
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 02/02: Merge branch 'master' of
github.com:apache/incubator-iotdb
Posted by hx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch fix_401_correct_chunk_size
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 99f9ef9e2390750425623159dad4d7d6768d1a4d
Merge: 6d65b74 6366f2b
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Mon Jan 6 21:47:36 2020 +0800
Merge branch 'master' of github.com:apache/incubator-iotdb
README.md | 60 +--
.../org/apache/iotdb/client/AbstractClient.java | 118 ++----
.../main/java/org/apache/iotdb/tool/ImportCsv.java | 7 +-
docs/Development-VoteRelease.md | 198 +++++++++
docs/Documentation/UserGuide/0-Content.md | 4 +-
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 9 +-
.../main/java/org/apache/iotdb/jdbc/Constant.java | 29 --
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 207 +--------
.../iotdb/jdbc/IoTDBMetadataResultMetadata.java | 193 ---------
.../apache/iotdb/jdbc/IoTDBMetadataResultSet.java | 427 -------------------
.../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 14 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 110 +----
.../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 463 ---------------------
.../jdbc/IoTDBMetadataResultMetadataTest.java | 108 -----
.../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 151 -------
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 79 +++-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 9 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 54 +--
.../iotdb/db/engine/flush/TsFileFlushPolicy.java | 54 +++
.../iotdb/db/engine/merge/task/MergeFileTask.java | 18 +
.../engine/storagegroup/StorageGroupProcessor.java | 89 ++--
.../db/engine/storagegroup/TsFileProcessor.java | 26 +-
.../db/engine/storagegroup/TsFileResource.java | 86 +++-
.../java/org/apache/iotdb/db/metadata/MGraph.java | 6 +-
.../org/apache/iotdb/db/metadata/MManager.java | 21 +-
.../java/org/apache/iotdb/db/metadata/MTree.java | 67 ++-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 7 +
.../qp/executor/AbstractQueryProcessExecutor.java | 187 +++++++--
.../db/qp/executor/IQueryProcessExecutor.java | 18 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 12 +-
.../db/qp/logical/crud/BasicFunctionOperator.java | 3 -
.../iotdb/db/qp/logical/crud/FilterOperator.java | 15 +-
.../sys/CountOperator.java} | 43 +-
.../sys/ShowChildPathsOperator.java} | 35 +-
.../sys/ShowDevicesOperator.java} | 35 +-
.../sys/ShowTimeSeriesOperator.java} | 35 +-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 24 +-
.../physical/sys/{ShowPlan.java => CountPlan.java} | 40 +-
.../sys/{ShowPlan.java => ShowChildPathsPlan.java} | 34 +-
.../sys/{ShowPlan.java => ShowDevicesPlan.java} | 36 +-
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 3 +-
.../iotdb/db/qp/physical/sys/ShowTTLPlan.java | 18 +-
.../sys/{ShowPlan.java => ShowTimeSeriesPlan.java} | 35 +-
.../iotdb/db/qp/strategy/LogicalGenerator.java | 68 ++-
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 132 +++---
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 131 +++++-
.../optimizer/MergeSingleFilterOptimizer.java | 13 +-
.../db/query/dataset/DeviceIterateDataSet.java | 4 +-
.../NewEngineDataSetWithoutValueFilter.java | 14 +-
.../iotdb/db/query/executor/EngineExecutor.java | 5 +-
.../iotdb/db/query/reader/ManagedSeriesReader.java | 37 ++
.../SeriesReaderWithoutValueFilter.java | 8 +-
.../org/apache/iotdb/db/service/StaticResps.java | 36 ++
.../org/apache/iotdb/db/service/TSServiceImpl.java | 119 ++----
.../iotdb/db/tools/TsFileResourcePrinter.java | 8 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 31 +-
.../writelog/recover/TsFileRecoverPerformer.java | 50 ++-
.../db/engine/cache/DeviceMetaDataCacheTest.java | 3 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 3 +
.../apache/iotdb/db/engine/merge/MergeTest.java | 3 +
.../storagegroup/StorageGroupProcessorTest.java | 3 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 6 +-
.../db/integration/IoTDBDeleteStorageGroupIT.java | 4 +-
.../db/integration/IoTDBFlushQueryMergeTest.java | 64 ++-
.../iotdb/db/integration/IoTDBGroupbyDeviceIT.java | 18 +-
.../integration/IoTDBLoadExternalTsfileTest.java | 2 +-
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 263 +++++++-----
.../iotdb/db/metadata/MManagerBasicTest.java | 35 +-
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 9 +-
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 12 +-
.../iotdb/db/query/reader/ReaderTestHelper.java | 3 +-
.../SeriesReaderWithValueFilterTest.java | 3 +-
.../SeriesReaderWithoutValueFilterTest.java | 5 +-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 2 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 62 ++-
.../writelog/recover/UnseqTsFileRecoverTest.java | 4 +-
service-rpc/rpc-changelist.md | 6 +
service-rpc/src/main/thrift/rpc.thrift | 10 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 2 +-
79 files changed, 1791 insertions(+), 2574 deletions(-)
[incubator-iotdb] 01/02: IOTDB-401,
skip the size of a chunk header if the chunk has no data points
Posted by hx...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch fix_401_correct_chunk_size
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 6d65b743d62a2600159b5cd1a246faa422565424
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Mon Jan 6 21:47:06 2020 +0800
IOTDB-401, skip the size of a chunk header if the chunk has no data points
---
.../apache/iotdb/db/tools/TsFileSketchTool.java | 11 +-
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 3 +
.../iotdb/tsfile/write/chunk/IChunkWriter.java | 2 +
.../iotdb/tsfile/write/TsFileWriterTest.java | 255 +++++++++++++++++++++
4 files changed, 269 insertions(+), 2 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index fbeba9d..2169498 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -130,8 +130,15 @@ public class TsFileSketchTool {
}
// metadata begins
- printlnBoth(pw, String.format("%20s", tsDeviceMetadataIndexSortedList.get(0).getOffset() - 1)
- + "|\t[marker] 2");
+ if (tsDeviceMetadataIndexSortedList.size() != 0) {
+ printlnBoth(pw,
+ String.format("%20s", tsDeviceMetadataIndexSortedList.get(0).getOffset() - 1)
+ + "|\t[marker] 2");
+ } else {
+ printlnBoth(pw,
+ String.format("%20s", reader.getFileMetadataPos() - 1)
+ + "|\t[marker] 2");
+ }
for (
int i = 0; i < tsDeviceMetadataSortedList.size(); i++) {
TsDeviceMetadata tsDeviceMetadata = tsDeviceMetadataSortedList.get(i);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java
index 486cf41..8d11612 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/ChunkWriterImpl.java
@@ -231,6 +231,9 @@ public class ChunkWriterImpl implements IChunkWriter {
@Override
public long getCurrentChunkSize() {
+ if (this.getCurrentDataSize() == 0) {
+ return 0;
+ }
// return the serialized size of the chunk header + all pages
return ChunkHeader.getSerializedSize(measurementSchema.getMeasurementId()) + this
.getCurrentDataSize();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java
index c375b45..9259387 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/chunk/IChunkWriter.java
@@ -103,6 +103,8 @@ public interface IChunkWriter {
* return the serialized size of the chunk header + all pages (not including the un-sealed page).
* Notice, call this method before calling writeToFileWriter(), otherwise the page buffer in
* memory will be cleared.
+ * <br> If there is no data points in the chunk, return 0 (i.e., in this case, the size of header
+ * is not calculated, because nothing will be serialized latter)</>
*/
long getCurrentChunkSize();
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileWriterTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileWriterTest.java
new file mode 100644
index 0000000..d23495c
--- /dev/null
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/TsFileWriterTest.java
@@ -0,0 +1,255 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile.write;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import org.apache.iotdb.tsfile.exception.encoding.TsFileEncodingException;
+import org.apache.iotdb.tsfile.exception.write.NoMeasurementException;
+import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.read.ReadOnlyTsFile;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.RowRecord;
+import org.apache.iotdb.tsfile.read.expression.QueryExpression;
+import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
+import org.apache.iotdb.tsfile.write.record.RowBatch;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.FloatDataPoint;
+import org.apache.iotdb.tsfile.write.record.datapoint.IntDataPoint;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TsFileWriterTest {
+ TsFileWriter writer = null;
+ long fileName = System.nanoTime();
+ boolean closed = false;
+ @Before
+ public void setUp() {
+ try {
+ writer = new TsFileWriter(new File("target/tsfileWriter-" + fileName));
+ addMeasurement();
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ @After
+ public void tearDown() {
+ if (!closed) {
+ closeFile();
+ }
+ try {
+ Files.deleteIfExists(Path.of("target/tsfileWriter-" + fileName));
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+
+ private void addMeasurement() {
+ try {
+ //String measurementId, TSDataType type, TSEncoding encoding,
+ // CompressionType compressionType
+ writer.addMeasurement(
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY));
+ } catch (WriteProcessException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ try {
+ //String measurementId, TSDataType type, TSEncoding encoding,
+ // CompressionType compressionType
+ writer.addMeasurement(
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY));
+ } catch (WriteProcessException e) {
+ Assert.assertEquals("given measurement has exists! s1", e.getMessage());
+ }
+ try {
+ //String measurementId, TSDataType type, TSEncoding encoding,
+ // CompressionType compressionType
+ writer.addMeasurement(
+ new MeasurementSchema("s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY));
+ } catch (WriteProcessException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+
+ }
+
+ @Test
+ public void addMeasurementTest() {
+ closeFile();
+ readNothing();
+ }
+
+
+ private void writeTSRecord() throws IOException, WriteProcessException {
+ //normal
+ TSRecord record = new TSRecord( 10000, "d1");
+ record.addTuple(new FloatDataPoint("s1", 5.0f));
+ record.addTuple(new IntDataPoint("s2", 5));
+ writer.write(record);
+
+ //not existed time series
+ record = new TSRecord( 10001, "d1");
+ record.addTuple(new FloatDataPoint("s3", 5.0f));
+ try {
+ writer.write(record);
+ } catch (WriteProcessException e) {
+ assertTrue(e instanceof NoMeasurementException);
+ }
+ }
+
+ @Test
+ public void writeTSRecordTest() throws IOException, WriteProcessException {
+ writeTSRecord();
+ closeFile();
+ readOneRow();
+ }
+
+ @Test
+ public void writeIncorrectTSRecord0() throws IOException, WriteProcessException {
+ //incorrect data type
+ TSRecord record = new TSRecord(10002, "d2");
+ record.addTuple(new IntDataPoint("s1", 5));
+ try {
+ writer.write(record);
+ } catch (TsFileEncodingException e) {
+ //do nothing
+ }
+ closeFile();
+ }
+
+ @Test
+ public void writeIncorrectTSRecord() throws IOException, WriteProcessException {
+ writeTSRecord();
+ //incorrect data type
+ TSRecord record = new TSRecord(10002, "d2");
+ record.addTuple(new IntDataPoint("s1", 5));
+ try {
+ writer.write(record);
+ } catch (TsFileEncodingException e) {
+ //do nothing
+ }
+ closeFile();
+ readOneRow();
+ }
+
+ @Test
+ public void writeRowBatch() throws IOException, WriteProcessException {
+ RowBatch rowBatch = new RowBatch("d1", Arrays.asList(new MeasurementSchema[]{
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY),
+ new MeasurementSchema("s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY)
+ }));
+ rowBatch.timestamps[0] = 10000;
+ ((float[])rowBatch.values[0])[0] = 5.0f;
+ ((int[])rowBatch.values[1])[0] = 5;
+ rowBatch.batchSize = 1;
+ writer.write(rowBatch);
+ closeFile();
+ readOneRow();
+ }
+
+ @Test
+ public void writeRowBatch2() throws IOException, WriteProcessException {
+ RowBatch rowBatch = new RowBatch("d1", Arrays.asList(new MeasurementSchema[]{
+ new MeasurementSchema("s1", TSDataType.FLOAT, TSEncoding.RLE, CompressionType.SNAPPY),
+ new MeasurementSchema("s2", TSDataType.INT32, TSEncoding.RLE, CompressionType.SNAPPY)
+ }));
+ rowBatch.timestamps[0] = 10000;
+ ((float[])rowBatch.values[0])[0] = 5.0f;
+ rowBatch.batchSize = 1;
+ writer.write(rowBatch);
+ closeFile();
+ //in this case, the value of s2 = 0 at time 10000.
+ }
+
+ @Test
+ public void getIOWriter() throws IOException {
+ //The interface is just for test
+ writer.getIOWriter();
+ }
+
+ @Test
+ public void flushForTest() throws IOException {
+ //The interface is just for test
+ writer.flushForTest();
+ }
+
+ private void closeFile() {
+ try {
+ closed = true;
+ writer.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ private void readNothing() {
+ //using TsFileReader for test
+ try {
+ ReadOnlyTsFile readOnlyTsFile = new ReadOnlyTsFile(
+ new TsFileSequenceReader("target/tsfileWriter-" + fileName));
+ QueryDataSet dataSet = readOnlyTsFile.query(QueryExpression.create()
+ .addSelectedPath(new org.apache.iotdb.tsfile.read.common.Path("d1.s1"))
+ .addSelectedPath(new org.apache.iotdb.tsfile.read.common.Path("d1.s2")));
+ assertFalse(dataSet.hasNext());
+ readOnlyTsFile.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+ private void readOneRow() {
+ try {
+ ReadOnlyTsFile readOnlyTsFile = new ReadOnlyTsFile(
+ new TsFileSequenceReader("target/tsfileWriter-" + fileName));
+ QueryDataSet dataSet = readOnlyTsFile.query(QueryExpression.create()
+ .addSelectedPath(new org.apache.iotdb.tsfile.read.common.Path("d1.s1"))
+ .addSelectedPath(new org.apache.iotdb.tsfile.read.common.Path("d1.s2"))
+ .addSelectedPath(new org.apache.iotdb.tsfile.read.common.Path("d1.s3")));
+ while(dataSet.hasNext()) {
+ RowRecord result = dataSet.next();
+ assertEquals(2, result.getFields().size());
+ assertEquals(10000, result.getTimestamp());
+ assertEquals(5.0f, result.getFields().get(0).getFloatV(), 0.00001);
+ assertEquals(5, result.getFields().get(1).getIntV());
+ }
+ readOnlyTsFile.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ fail(e.getMessage());
+ }
+ }
+}