You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xu...@apache.org on 2021/12/06 01:21:13 UTC
[iotdb] branch tsfile_test_xkf created (now 704f5b9)
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a change to branch tsfile_test_xkf
in repository https://gitbox.apache.org/repos/asf/iotdb.git.
at 704f5b9 for performance test
This branch includes the following new commits:
new 704f5b9 for performance test
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[iotdb] 01/01: for performance test
Posted by xu...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a commit to branch tsfile_test_xkf
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 704f5b99d65c20f3f469d676644b30a9d03f458e
Author: 151250176 <15...@smail.nju.edu.cn>
AuthorDate: Mon Dec 6 09:21:48 2021 +0800
for performance test
---
.../main/java/org/apache/iotdb/SessionExample.java | 57 +--
example/tsfile/pom.xml | 32 ++
.../java/org/apache/iotdb/tsfile/Constant.java | 2 +
.../org/apache/iotdb/tsfile/TsFileRawRead.java | 67 +++
.../java/org/apache/iotdb/tsfile/TsFileRead.java | 50 +--
.../org/apache/iotdb/tsfile/TsFileWriteTest.java | 121 ++++++
.../engine/storagegroup/StorageGroupProcessor.java | 15 +
.../file/metadata/statistics/Statistics.java | 68 +--
.../iotdb/tsfile/read/TsFileSequenceReader.java | 30 +-
.../org/apache/iotdb/tsfile/write/PerfTest.java | 479 ++++++++++-----------
10 files changed, 591 insertions(+), 330 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index 1f1c164..3ac6c72 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -73,9 +73,9 @@ public class SessionExample {
}
// createTemplate();
- createTimeseries();
- createMultiTimeseries();
- insertRecord();
+ // createTimeseries();
+ // createMultiTimeseries();
+ // insertRecord();
insertTablet();
// insertTabletWithNullValues();
// insertTablets();
@@ -99,9 +99,9 @@ public class SessionExample {
// set session fetchSize
sessionEnableRedirect.setFetchSize(10000);
- insertRecord4Redirect();
- query4Redirect();
- sessionEnableRedirect.close();
+ // insertRecord4Redirect();
+ // query4Redirect();
+ // sessionEnableRedirect.close();
session.close();
}
@@ -382,8 +382,9 @@ public class SessionExample {
// Method 1 to add tablet data
long timestamp = System.currentTimeMillis();
+ long curTime = System.nanoTime();
- for (long row = 0; row < 100; row++) {
+ for (long row = 0; row < 1000000; row++) {
int rowIndex = tablet.rowSize++;
tablet.addTimestamp(rowIndex, timestamp);
for (int s = 0; s < 3; s++) {
@@ -402,27 +403,31 @@ public class SessionExample {
tablet.reset();
}
- // Method 2 to add tablet data
- long[] timestamps = tablet.timestamps;
- Object[] values = tablet.values;
+ long finishTime = System.nanoTime();
- for (long time = 0; time < 100; time++) {
- int row = tablet.rowSize++;
- timestamps[row] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor = (long[]) values[i];
- sensor[row] = i;
- }
- if (tablet.rowSize == tablet.getMaxRowNumber()) {
- session.insertTablet(tablet, true);
- tablet.reset();
- }
- }
+ System.out.println("write finished time: " + (finishTime - curTime) / 1000 / 1000 + " ms.");
- if (tablet.rowSize != 0) {
- session.insertTablet(tablet);
- tablet.reset();
- }
+ // Method 2 to add tablet data
+ // long[] timestamps = tablet.timestamps;
+ // Object[] values = tablet.values;
+ //
+ // for (long time = 0; time < 100; time++) {
+ // int row = tablet.rowSize++;
+ // timestamps[row] = time;
+ // for (int i = 0; i < 3; i++) {
+ // long[] sensor = (long[]) values[i];
+ // sensor[row] = i;
+ // }
+ // if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ // session.insertTablet(tablet, true);
+ // tablet.reset();
+ // }
+ // }
+ //
+ // if (tablet.rowSize != 0) {
+ // session.insertTablet(tablet);
+ // tablet.reset();
+ // }
}
private static void insertTabletWithNullValues()
diff --git a/example/tsfile/pom.xml b/example/tsfile/pom.xml
index 13a53a9..304e390 100644
--- a/example/tsfile/pom.xml
+++ b/example/tsfile/pom.xml
@@ -35,5 +35,37 @@
<artifactId>tsfile</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ </dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.5.5</version>
+ <executions>
+ <execution>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.iotdb.tsfile.TsFileWriteTest</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ <id>make-assembly</id>
+ <phase>package</phase>
+ <goals>
+ <goal>single</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</project>
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/Constant.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/Constant.java
index 09f4213..15f9cfe 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/Constant.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/Constant.java
@@ -21,6 +21,8 @@ package org.apache.iotdb.tsfile;
public class Constant {
+ public static final String SENSOR_ = "sensor_";
+
private Constant() {}
static final String SENSOR_1 = "sensor_1";
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileRawRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileRawRead.java
new file mode 100644
index 0000000..f7318c5
--- /dev/null
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileRawRead.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.tsfile;
+
+import org.apache.iotdb.tsfile.read.TsFileReader;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.apache.iotdb.tsfile.read.common.Path;
+
+import java.io.IOException;
+
+public class TsFileRawRead {
+
+ private static final String DEVICE1 = "device_1";
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int treeType; // 0=Zesong Tree, 1=B+ Tree
+ public static int fileNum;
+
+ public static void main(String[] args) throws IOException {
+ try {
+ deviceNum = 10000; // Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = 100; // Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = 1; // Integer.parseInt(cl.getOptionValue("f"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ long totalStartTime = System.nanoTime();
+ for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
+ // file path
+ String path =
+ "/Users/SilverNarcissus/iotdb/tsfile_test"
+ + "/withoutStatAndMeasurement_1/"
+ + deviceNum
+ + "."
+ + sensorNum
+ + "/test"
+ + fileIndex
+ + ".tsfile";
+
+ // raw data query
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ TsFileReader readTsFile = new TsFileReader(reader)) {
+ reader.readTimeseriesMetadata(new Path("device_1", "sensor_1"), true);
+ }
+ }
+ long totalTime = (System.nanoTime() - totalStartTime) / 1000_000;
+ System.out.println("Total raw read cost time: " + totalTime + "ms");
+ System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms");
+ }
+}
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java
index 9ce88fc..b3687c3 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileRead.java
@@ -23,11 +23,6 @@ import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.read.expression.IExpression;
import org.apache.iotdb.tsfile.read.expression.QueryExpression;
-import org.apache.iotdb.tsfile.read.expression.impl.BinaryExpression;
-import org.apache.iotdb.tsfile.read.expression.impl.GlobalTimeExpression;
-import org.apache.iotdb.tsfile.read.expression.impl.SingleSeriesExpression;
-import org.apache.iotdb.tsfile.read.filter.TimeFilter;
-import org.apache.iotdb.tsfile.read.filter.ValueFilter;
import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import java.io.IOException;
@@ -55,7 +50,7 @@ public class TsFileRead {
public static void main(String[] args) throws IOException {
// file path
- String path = "test.tsfile";
+ String path = "/Users/SilverNarcissus/iotdb/tsfile_test0/1.1/test0.tsfile";
// create reader and get the readTsFile interface
try (TsFileSequenceReader reader = new TsFileSequenceReader(path);
@@ -64,33 +59,32 @@ public class TsFileRead {
// use these paths(all measurements) for all the queries
ArrayList<Path> paths = new ArrayList<>();
paths.add(new Path(DEVICE1, "sensor_1"));
- paths.add(new Path(DEVICE1, "sensor_2"));
- paths.add(new Path(DEVICE1, "sensor_3"));
// no filter, should select 1 2 3 4 6 7 8
queryAndPrint(paths, readTsFile, null);
// time filter : 4 <= time <= 10, should select 4 6 7 8
- IExpression timeFilter =
- BinaryExpression.and(
- new GlobalTimeExpression(TimeFilter.gtEq(4L)),
- new GlobalTimeExpression(TimeFilter.ltEq(10L)));
- queryAndPrint(paths, readTsFile, timeFilter);
-
- // value filter : device_1.sensor_2 <= 20, should select 1 2 4 6 7
- IExpression valueFilter =
- new SingleSeriesExpression(new Path(DEVICE1, "sensor_2"), ValueFilter.ltEq(20L));
- queryAndPrint(paths, readTsFile, valueFilter);
-
- // time filter : 4 <= time <= 10, value filter : device_1.sensor_3 >= 20, should select 4 7 8
- timeFilter =
- BinaryExpression.and(
- new GlobalTimeExpression(TimeFilter.gtEq(4L)),
- new GlobalTimeExpression(TimeFilter.ltEq(10L)));
- valueFilter =
- new SingleSeriesExpression(new Path(DEVICE1, "sensor_3"), ValueFilter.gtEq(20L));
- IExpression finalFilter = BinaryExpression.and(timeFilter, valueFilter);
- queryAndPrint(paths, readTsFile, finalFilter);
+ // IExpression timeFilter =
+ // BinaryExpression.and(
+ // new GlobalTimeExpression(TimeFilter.gtEq(4L)),
+ // new GlobalTimeExpression(TimeFilter.ltEq(10L)));
+ // queryAndPrint(paths, readTsFile, timeFilter);
+ //
+ // // value filter : device_1.sensor_2 <= 20, should select 1 2 4 6 7
+ // IExpression valueFilter =
+ // new SingleSeriesExpression(new Path(DEVICE1, "sensor_2"), ValueFilter.ltEq(20L));
+ // queryAndPrint(paths, readTsFile, valueFilter);
+ //
+ // // time filter : 4 <= time <= 10, value filter : device_1.sensor_3 >= 20, should
+ // select 4 7 8
+ // timeFilter =
+ // BinaryExpression.and(
+ // new GlobalTimeExpression(TimeFilter.gtEq(4L)),
+ // new GlobalTimeExpression(TimeFilter.ltEq(10L)));
+ // valueFilter =
+ // new SingleSeriesExpression(new Path(DEVICE1, "sensor_3"), ValueFilter.gtEq(20L));
+ // IExpression finalFilter = BinaryExpression.and(timeFilter, valueFilter);
+ // queryAndPrint(paths, readTsFile, finalFilter);
}
}
}
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteTest.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteTest.java
new file mode 100644
index 0000000..3106825
--- /dev/null
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/TsFileWriteTest.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.tsfile;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.TSRecord;
+import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
+import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
+import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
+
+import java.io.File;
+import java.util.Random;
+
+/**
+ * An example of writing data with TSRecord to TsFile It uses the interface: public void
+ * addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException
+ */
+public class TsFileWriteTest {
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int fileNum;
+ public static int pointNum = 10;
+
+ public static void main(String[] args) {
+ try {
+ deviceNum = 10000; // Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = 100; // Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = 1; // Integer.parseInt(cl.getOptionValue("f"));
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+
+ for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
+ try {
+ String path =
+ "/home/fit/xkf"
+ + "/without/"
+ + deviceNum
+ + "."
+ + sensorNum
+ + "/test"
+ + fileIndex
+ + ".tsfile";
+ File f = FSFactoryProducer.getFSFactory().getFile(path);
+ if (f.exists()) {
+ f.delete();
+ }
+
+ try {
+ TsFileWriter tsFileWriter = new TsFileWriter(f);
+ for (int i = 1; i <= deviceNum; i++) {
+ for (int j = 1; j <= sensorNum; j++) {
+ Path path1 = new Path(Constant.DEVICE_PREFIX + i);
+ tsFileWriter.registerTimeseries(
+ path1,
+ new UnaryMeasurementSchema(
+ Constant.SENSOR_ + j,
+ TSDataType.INT64,
+ TSEncoding.RLE,
+ CompressionType.GZIP));
+ }
+ }
+
+ long curTime = System.nanoTime();
+ // construct TSRecord
+ int count = 0;
+ for (int j = 1; j <= deviceNum; j++) {
+ for (int i = 1; i <= pointNum; i++) {
+ TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + j);
+ for (int t = 1; t <= sensorNum; t++) {
+ DataPoint dPoint1 =
+ new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong());
+ tsRecord.addTuple(dPoint1);
+ count++;
+ }
+ // write TSRecord
+ tsFileWriter.write(tsRecord);
+ if (count % 100000 == 0) {
+ System.out.println(count);
+ tsFileWriter.flushAllChunkGroups();
+ }
+ }
+ }
+ tsFileWriter.close();
+ long finishTime = System.nanoTime();
+
+ System.out.println(
+ "write finished time: " + (finishTime - curTime) / 1000 / 1000 + " ms.");
+ } catch (Throwable e) {
+ e.printStackTrace();
+ System.out.println(e.getMessage());
+ }
+ } catch (Throwable e) {
+ e.printStackTrace();
+ System.out.println(e.getMessage());
+ }
+ }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 43e3154..096bc7e 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@ -93,6 +93,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.MappedByteBuffer;
import java.nio.file.Files;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -290,6 +292,8 @@ public class StorageGroupProcessor {
public static final long COMPACTION_TASK_SUBMIT_DELAY = 20L * 1000L;
+ private static MessageDigest md;
+
/** get the direct byte buffer from pool, each fetch contains two ByteBuffer */
public ByteBuffer[] getWalDirectByteBuffer() {
ByteBuffer[] res = new ByteBuffer[2];
@@ -406,6 +410,11 @@ public class StorageGroupProcessor {
config.getWalPoolTrimIntervalInMS(),
TimeUnit.MILLISECONDS);
recover();
+ try {
+ md = MessageDigest.getInstance("SHA-256");
+ } catch (NoSuchAlgorithmException e) {
+ e.printStackTrace();
+ }
}
public String getLogicalStorageGroupName() {
@@ -937,6 +946,12 @@ public class StorageGroupProcessor {
throws BatchProcessException, TriggerExecutionException {
writeLock("insertTablet");
+
+ // for test
+ String deviceID = insertTabletPlan.getDeviceId().toString();
+ md.digest(deviceID.getBytes());
+ md.reset();
+
try {
TSStatus[] results = new TSStatus[insertTabletPlan.getRowCount()];
Arrays.fill(results, RpcUtils.SUCCESS_STATUS);
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
index abec0e7..d6579b1 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/statistics/Statistics.java
@@ -106,6 +106,28 @@ public abstract class Statistics<T extends Serializable> {
}
}
+ public static Statistics<? extends Serializable> deserialize(
+ InputStream inputStream, TSDataType dataType) throws IOException {
+ Statistics<? extends Serializable> statistics = getStatsByType(dataType);
+ statistics.setCount(ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream));
+ statistics.setStartTime(ReadWriteIOUtils.readLong(inputStream));
+ statistics.setEndTime(ReadWriteIOUtils.readLong(inputStream));
+ statistics.deserialize(inputStream);
+ statistics.isEmpty = false;
+ return statistics;
+ }
+
+ public static Statistics<? extends Serializable> deserialize(
+ ByteBuffer buffer, TSDataType dataType) {
+ Statistics<? extends Serializable> statistics = getStatsByType(dataType);
+ statistics.setCount(ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
+ statistics.setStartTime(ReadWriteIOUtils.readLong(buffer));
+ statistics.setEndTime(ReadWriteIOUtils.readLong(buffer));
+ statistics.deserialize(buffer);
+ statistics.isEmpty = false;
+ return statistics;
+ }
+
public abstract TSDataType getType();
public int getSerializedSize() {
@@ -118,11 +140,11 @@ public abstract class Statistics<T extends Serializable> {
public int serialize(OutputStream outputStream) throws IOException {
int byteLen = 0;
- byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream);
- byteLen += ReadWriteIOUtils.write(startTime, outputStream);
- byteLen += ReadWriteIOUtils.write(endTime, outputStream);
+ // byteLen += ReadWriteForEncodingUtils.writeUnsignedVarInt(count, outputStream);
+ // byteLen += ReadWriteIOUtils.write(startTime, outputStream);
+ // byteLen += ReadWriteIOUtils.write(endTime, outputStream);
// value statistics of different data type
- byteLen += serializeStats(outputStream);
+ // byteLen += serializeStats(outputStream);
return byteLen;
}
@@ -343,48 +365,26 @@ public abstract class Statistics<T extends Serializable> {
throw new UnsupportedOperationException();
}
- public static Statistics<? extends Serializable> deserialize(
- InputStream inputStream, TSDataType dataType) throws IOException {
- Statistics<? extends Serializable> statistics = getStatsByType(dataType);
- statistics.setCount(ReadWriteForEncodingUtils.readUnsignedVarInt(inputStream));
- statistics.setStartTime(ReadWriteIOUtils.readLong(inputStream));
- statistics.setEndTime(ReadWriteIOUtils.readLong(inputStream));
- statistics.deserialize(inputStream);
- statistics.isEmpty = false;
- return statistics;
- }
-
- public static Statistics<? extends Serializable> deserialize(
- ByteBuffer buffer, TSDataType dataType) {
- Statistics<? extends Serializable> statistics = getStatsByType(dataType);
- statistics.setCount(ReadWriteForEncodingUtils.readUnsignedVarInt(buffer));
- statistics.setStartTime(ReadWriteIOUtils.readLong(buffer));
- statistics.setEndTime(ReadWriteIOUtils.readLong(buffer));
- statistics.deserialize(buffer);
- statistics.isEmpty = false;
- return statistics;
- }
-
public long getStartTime() {
return startTime;
}
- public long getEndTime() {
- return endTime;
- }
-
- public long getCount() {
- return count;
- }
-
public void setStartTime(long startTime) {
this.startTime = startTime;
}
+ public long getEndTime() {
+ return endTime;
+ }
+
public void setEndTime(long endTime) {
this.endTime = endTime;
}
+ public long getCount() {
+ return count;
+ }
+
public void setCount(int count) {
this.count = count;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index 6f5e088..009242c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -27,7 +27,15 @@ import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.header.ChunkHeader;
import org.apache.iotdb.tsfile.file.header.PageHeader;
-import org.apache.iotdb.tsfile.file.metadata.*;
+import org.apache.iotdb.tsfile.file.metadata.AlignedTimeSeriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
+import org.apache.iotdb.tsfile.file.metadata.ITimeSeriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry;
+import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode;
+import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
@@ -54,7 +62,20 @@ import java.io.IOException;
import java.io.Serializable;
import java.nio.BufferOverflowException;
import java.nio.ByteBuffer;
-import java.util.*;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+import java.util.Queue;
+import java.util.Set;
+import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -335,6 +356,11 @@ public class TsFileSequenceReader implements AutoCloseable {
if (metadataIndexPair == null) {
return null;
}
+
+ System.out.println(fileSize());
+ System.out.println(metadataIndexPair.right);
+ tsFileInput.close();
+ System.exit(-1);
List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
buffer = readData(metadataIndexPair.left.getOffset(), metadataIndexPair.right);
while (buffer.hasRemaining()) {
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java
index 8a46d95..4e4d850 100755
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/PerfTest.java
@@ -1,240 +1,239 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.tsfile.write;
-
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
-import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
-import org.apache.iotdb.tsfile.constant.TestConstant;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.utils.RecordUtils;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.schema.Schema;
-import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
-
-import ch.qos.logback.classic.Level;
-import ch.qos.logback.classic.LoggerContext;
-import com.google.gson.JsonObject;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.util.Random;
-import java.util.Scanner;
-
-/**
- * This is used for performance test, no asserting. User could change {@code ROW_COUNT} for larger
- * data test.
- */
-public class PerfTest {
-
- public static final int ROW_COUNT = 1000; // 0000;
- private static final Logger LOG = LoggerFactory.getLogger(PerfTest.class);
- public static TsFileWriter innerWriter;
- public static String inputDataFile;
- public static String outputDataFile;
- public static String errorOutputDataFile;
- public static Schema schema;
- public static Random rm = new Random();
-
- private static void generateSampleInputDataFile() throws IOException {
- File file = new File(inputDataFile);
- if (file.exists()) {
- file.delete();
- }
- FileWriter fw = new FileWriter(file);
-
- long startTime = System.currentTimeMillis();
- startTime = startTime - startTime % 1000;
- try {
- for (int i = 0; i < ROW_COUNT; i++) {
- String string4 = ",s4," + (char) (97 + i % 26);
- // write d1
- String d1 =
- "d1," + (startTime + i) + ",s1," + (i * 10 + 1) + ",s2," + (i * 10 + 2) + string4;
- if (rm.nextInt(1000) < 100) {
- // LOG.info("write null to d1:" + (startTime + i));
- d1 = "d1," + (startTime + i) + ",s1,,s2," + (i * 10 + 2) + string4;
- }
- if (i % 5 == 0) {
- d1 += ",s3," + (i * 10 + 3);
- }
- fw.write(d1 + "\r\n");
-
- // write d2
- String d2 =
- "d2," + (startTime + i) + ",s2," + (i * 10 + 2) + ",s3," + (i * 10 + 3) + string4;
- if (rm.nextInt(1000) < 100) {
- // LOG.info("write null to d2:" + (startTime + i));
- d2 = "d2," + (startTime + i) + ",s2,,s3," + (i * 10 + 3) + string4;
- }
- if (i % 5 == 0) {
- d2 += ",s1," + (i * 10 + 1);
- }
- fw.write(d2 + "\r\n");
- }
- // write error
- String d =
- "d2,3,"
- + (startTime + ROW_COUNT)
- + ",s2,"
- + (ROW_COUNT * 10 + 2)
- + ",s3,"
- + (ROW_COUNT * 10 + 3);
- fw.write(d + "\r\n");
- d = "d2," + (startTime + ROW_COUNT + 1) + ",2,s-1," + (ROW_COUNT * 10 + 2);
- fw.write(d + "\r\n");
- } finally {
- fw.close();
- }
- }
-
- private static void write() throws IOException, InterruptedException {
- File file = new File(outputDataFile);
- File errorFile = new File(errorOutputDataFile);
- if (file.exists()) {
- file.delete();
- }
- if (errorFile.exists()) {
- errorFile.delete();
- }
-
- // TSFileDescriptor.conf.chunkGroupSize = 2000;
- // TSFileDescriptor.conf.pageSizeInByte = 100;
- innerWriter = new TsFileWriter(file, schema, TSFileDescriptor.getInstance().getConfig());
-
- // write
- try {
- writeToFile(schema);
- } catch (WriteProcessException e) {
- e.printStackTrace();
- }
- LOG.info("write to file successfully!!");
- }
-
- private static Scanner getDataFile(String path) {
- File file = new File(path);
- try {
- Scanner in = new Scanner(file);
- return in;
- } catch (FileNotFoundException e) {
- e.printStackTrace();
- return null;
- }
- }
-
- private static void writeToFile(Schema schema)
- throws InterruptedException, IOException, WriteProcessException {
- Scanner in = getDataFile(inputDataFile);
- assert in != null;
- while (in.hasNextLine()) {
- String str = in.nextLine();
- TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
- innerWriter.write(record);
- }
- innerWriter.close();
- }
-
- private static Schema generateTestData() {
- Schema schema = new Schema();
- TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
- schema.registerTimeseries(
- new Path("d1"),
- new UnaryMeasurementSchema(
- "s1", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
- schema.registerTimeseries(
- new Path("d1"),
- new UnaryMeasurementSchema(
- "s2", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
- schema.registerTimeseries(
- new Path("d1"),
- new UnaryMeasurementSchema(
- "s3", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
- schema.registerTimeseries(
- new Path("d1"), new UnaryMeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN));
- schema.registerTimeseries(
- new Path("d2"),
- new UnaryMeasurementSchema(
- "s1", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
- schema.registerTimeseries(
- new Path("d2"),
- new UnaryMeasurementSchema(
- "s2", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
- schema.registerTimeseries(
- new Path("d2"),
- new UnaryMeasurementSchema(
- "s3", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
- schema.registerTimeseries(
- new Path("d2"), new UnaryMeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN));
-
- JsonObject s4 = new JsonObject();
- s4.addProperty(JsonFormatConstant.MEASUREMENT_UID, "s4");
- s4.addProperty(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
- s4.addProperty(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
- return schema;
- }
-
- @Before
- public void prepare() throws IOException {
- LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
- // set global log level
- ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root");
- logger.setLevel(Level.toLevel("info"));
-
- inputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("perTestInputData");
- outputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("perTestOutputData.tsfile");
- errorOutputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("perTestErrorOutputData.tsfile");
- schema = generateTestData();
- generateSampleInputDataFile();
- }
-
- @After
- public void after() {
- File file = new File(inputDataFile);
- if (file.exists()) {
- file.delete();
- }
- file = new File(outputDataFile);
- if (file.exists()) {
- file.delete();
- }
- file = new File(errorOutputDataFile);
- if (file.exists()) {
- file.delete();
- }
- LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
- // set global log level
- ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root");
- logger.setLevel(Level.toLevel("info"));
- }
-
- @Test
- public void writeTest() throws IOException, InterruptedException, WriteProcessException {
- write();
- }
-}
+/// *
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+// package org.apache.iotdb.tsfile.write;
+//
+// import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+// import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
+// import org.apache.iotdb.tsfile.common.constant.JsonFormatConstant;
+// import org.apache.iotdb.tsfile.constant.TestConstant;
+// import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
+// import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+// import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+// import org.apache.iotdb.tsfile.read.common.Path;
+// import org.apache.iotdb.tsfile.utils.RecordUtils;
+// import org.apache.iotdb.tsfile.write.record.TSRecord;
+// import org.apache.iotdb.tsfile.write.schema.Schema;
+// import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
+//
+// import ch.qos.logback.classic.Level;
+// import ch.qos.logback.classic.LoggerContext;
+// import org.junit.After;
+// import org.junit.Before;
+// import org.junit.Test;
+// import org.slf4j.Logger;
+// import org.slf4j.LoggerFactory;
+//
+// import java.io.File;
+// import java.io.FileNotFoundException;
+// import java.io.FileWriter;
+// import java.io.IOException;
+// import java.util.Random;
+// import java.util.Scanner;
+//
+/// **
+// * This is used for performance test, no asserting. User could change {@code ROW_COUNT} for larger
+// * data test.
+// */
+// public class PerfTest {
+//
+// public static final int ROW_COUNT = 1000; // 0000;
+// private static final Logger LOG = LoggerFactory.getLogger(PerfTest.class);
+// public static TsFileWriter innerWriter;
+// public static String inputDataFile;
+// public static String outputDataFile;
+// public static String errorOutputDataFile;
+// public static Schema schema;
+// public static Random rm = new Random();
+//
+// private static void generateSampleInputDataFile() throws IOException {
+// File file = new File(inputDataFile);
+// if (file.exists()) {
+// file.delete();
+// }
+// FileWriter fw = new FileWriter(file);
+//
+// long startTime = System.currentTimeMillis();
+// startTime = startTime - startTime % 1000;
+// try {
+// for (int i = 0; i < ROW_COUNT; i++) {
+// String string4 = ",s4," + (char) (97 + i % 26);
+// // write d1
+// String d1 =
+// "d1," + (startTime + i) + ",s1," + (i * 10 + 1) + ",s2," + (i * 10 + 2) + string4;
+// if (rm.nextInt(1000) < 100) {
+// // LOG.info("write null to d1:" + (startTime + i));
+// d1 = "d1," + (startTime + i) + ",s1,,s2," + (i * 10 + 2) + string4;
+// }
+// if (i % 5 == 0) {
+// d1 += ",s3," + (i * 10 + 3);
+// }
+// fw.write(d1 + "\r\n");
+//
+// // write d2
+// String d2 =
+// "d2," + (startTime + i) + ",s2," + (i * 10 + 2) + ",s3," + (i * 10 + 3) + string4;
+// if (rm.nextInt(1000) < 100) {
+// // LOG.info("write null to d2:" + (startTime + i));
+// d2 = "d2," + (startTime + i) + ",s2,,s3," + (i * 10 + 3) + string4;
+// }
+// if (i % 5 == 0) {
+// d2 += ",s1," + (i * 10 + 1);
+// }
+// fw.write(d2 + "\r\n");
+// }
+// // write error
+// String d =
+// "d2,3,"
+// + (startTime + ROW_COUNT)
+// + ",s2,"
+// + (ROW_COUNT * 10 + 2)
+// + ",s3,"
+// + (ROW_COUNT * 10 + 3);
+// fw.write(d + "\r\n");
+// d = "d2," + (startTime + ROW_COUNT + 1) + ",2,s-1," + (ROW_COUNT * 10 + 2);
+// fw.write(d + "\r\n");
+// } finally {
+// fw.close();
+// }
+// }
+//
+// private static void write() throws IOException, InterruptedException {
+// File file = new File(outputDataFile);
+// File errorFile = new File(errorOutputDataFile);
+// if (file.exists()) {
+// file.delete();
+// }
+// if (errorFile.exists()) {
+// errorFile.delete();
+// }
+//
+// // TSFileDescriptor.conf.chunkGroupSize = 2000;
+// // TSFileDescriptor.conf.pageSizeInByte = 100;
+// innerWriter = new TsFileWriter(file, schema, TSFileDescriptor.getInstance().getConfig());
+//
+// // write
+// try {
+// writeToFile(schema);
+// } catch (WriteProcessException e) {
+// e.printStackTrace();
+// }
+// LOG.info("write to file successfully!!");
+// }
+//
+// private static Scanner getDataFile(String path) {
+// File file = new File(path);
+// try {
+// Scanner in = new Scanner(file);
+// return in;
+// } catch (FileNotFoundException e) {
+// e.printStackTrace();
+// return null;
+// }
+// }
+//
+// private static void writeToFile(Schema schema)
+// throws InterruptedException, IOException, WriteProcessException {
+// Scanner in = getDataFile(inputDataFile);
+// assert in != null;
+// while (in.hasNextLine()) {
+// String str = in.nextLine();
+// TSRecord record = RecordUtils.parseSimpleTupleRecord(str, schema);
+// innerWriter.write(record);
+// }
+// innerWriter.close();
+// }
+//
+// private static Schema generateTestData() {
+// Schema schema = new Schema();
+// TSFileConfig conf = TSFileDescriptor.getInstance().getConfig();
+// schema.registerTimeseries(
+// new Path("d1"),
+// new UnaryMeasurementSchema(
+// "s1", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
+// schema.registerTimeseries(
+// new Path("d1"),
+// new UnaryMeasurementSchema(
+// "s2", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
+// schema.registerTimeseries(
+// new Path("d1"),
+// new UnaryMeasurementSchema(
+// "s3", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
+// schema.registerTimeseries(
+// new Path("d1"), new UnaryMeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN));
+// schema.registerTimeseries(
+// new Path("d2"),
+// new UnaryMeasurementSchema(
+// "s1", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
+// schema.registerTimeseries(
+// new Path("d2"),
+// new UnaryMeasurementSchema(
+// "s2", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
+// schema.registerTimeseries(
+// new Path("d2"),
+// new UnaryMeasurementSchema(
+// "s3", TSDataType.INT64, TSEncoding.valueOf(conf.getValueEncoder())));
+// schema.registerTimeseries(
+// new Path("d2"), new UnaryMeasurementSchema("s4", TSDataType.TEXT, TSEncoding.PLAIN));
+//
+// JsonObject s4 = new JsonObject();
+// s4.addProperty(JsonFormatConstant.MEASUREMENT_UID, "s4");
+// s4.addProperty(JsonFormatConstant.DATA_TYPE, TSDataType.TEXT.toString());
+// s4.addProperty(JsonFormatConstant.MEASUREMENT_ENCODING, TSEncoding.PLAIN.toString());
+// return schema;
+// }
+//
+// @Before
+// public void prepare() throws IOException {
+// LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+// // set global log level
+// ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root");
+// logger.setLevel(Level.toLevel("info"));
+//
+// inputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("perTestInputData");
+// outputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("perTestOutputData.tsfile");
+// errorOutputDataFile = TestConstant.BASE_OUTPUT_PATH.concat("perTestErrorOutputData.tsfile");
+// schema = generateTestData();
+// generateSampleInputDataFile();
+// }
+//
+// @After
+// public void after() {
+// File file = new File(inputDataFile);
+// if (file.exists()) {
+// file.delete();
+// }
+// file = new File(outputDataFile);
+// if (file.exists()) {
+// file.delete();
+// }
+// file = new File(errorOutputDataFile);
+// if (file.exists()) {
+// file.delete();
+// }
+// LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory();
+// // set global log level
+// ch.qos.logback.classic.Logger logger = loggerContext.getLogger("root");
+// logger.setLevel(Level.toLevel("info"));
+// }
+//
+// @Test
+// public void writeTest() throws IOException, InterruptedException, WriteProcessException {
+// write();
+// }
+// }