You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by su...@apache.org on 2021/11/18 01:52:25 UTC
[iotdb] 03/03: fix bugs
This is an automated email from the ASF dual-hosted git repository.
sunzesong pushed a commit to branch experimental/index
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 8cf5af772b20dda505e954ed6dfbdfb810a2ef04
Author: samperson1997 <sz...@mails.tsinghua.edu.cn>
AuthorDate: Thu Nov 18 09:30:30 2021 +0800
fix bugs
---
example/tsfile/pom.xml | 2 +-
.../iotdb/tsfile/test1835/TsFileAggregation.java | 45 ++-
.../iotdb/tsfile/test1835/TsFileAggregationV2.java | 47 ++-
.../iotdb/tsfile/test1835/TsFileRawRead.java | 47 ++-
.../iotdb/tsfile/test1835/TsFileRawReadV2.java | 51 ++-
.../iotdb/tsfile/test1835/TsFileSketchToolV2.java | 425 ---------------------
.../apache/iotdb/tsfile/test1835/TsFileWrite.java | 67 ++--
.../{TsFileWrite.java => TsFileWriteV2.java} | 69 ++--
.../iotdb/tsfile/read/TsFileSequenceReader.java | 155 ++++----
.../tsfile/read/query/executor/TsFileExecutor.java | 3 +-
.../apache/iotdb/tsfile/write/TsFileWriter.java | 2 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 169 ++++----
12 files changed, 375 insertions(+), 707 deletions(-)
diff --git a/example/tsfile/pom.xml b/example/tsfile/pom.xml
index ce21451..45a9e33 100644
--- a/example/tsfile/pom.xml
+++ b/example/tsfile/pom.xml
@@ -51,7 +51,7 @@
<configuration>
<archive>
<manifest>
- <mainClass>org.apache.iotdb.tsfile.test1831.TsFileAggregation</mainClass>
+ <mainClass>org.apache.iotdb.tsfile.test1835.TsFileAggregationV2</mainClass>
</manifest>
</archive>
<descriptorRefs>
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java
index b4de97d..343d2ce 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregation.java
@@ -24,30 +24,38 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import java.io.IOException;
public class TsFileAggregation {
- private static final String DEVICE1 = "device_";
- public static int chunkNum;
- public static int deviceNum = 1;
- public static int sensorNum = 1;
- public static int fileNum = 1;
+ private static final String DEVICE1 = "device_1";
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int fileNum;
public static void main(String[] args) throws IOException {
- long costTime = 0L;
Options opts = new Options();
- // Option chunkNumOption =
- // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c");
- // opts.addOption(chunkNumOption);
+ Option deviceNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d");
+ opts.addOption(deviceNumOption);
+ Option sensorNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m");
+ opts.addOption(sensorNumOption);
+ Option fileNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f");
+ opts.addOption(fileNumOption);
BasicParser parser = new BasicParser();
CommandLine cl;
try {
cl = parser.parse(opts, args);
- // chunkNum = Integer.parseInt(cl.getOptionValue("c"));
+ deviceNum = Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = Integer.parseInt(cl.getOptionValue("f"));
} catch (Exception e) {
e.printStackTrace();
}
@@ -56,22 +64,23 @@ public class TsFileAggregation {
for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
// file path
String path =
- "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/"
+ "/data/szs/data/data/sequence/root.sg/1/"
+ deviceNum
- + "/test1.tsfile";
+ + "."
+ + sensorNum
+ + "/test"
+ + fileIndex
+ + ".tsfile";
// aggregation query
try (TsFileSequenceReader reader = new TsFileSequenceReader(path)) {
Path seriesPath = new Path(DEVICE1, "sensor_1");
- long startTime = System.nanoTime();
TimeseriesMetadata timeseriesMetadata = reader.readTimeseriesMetadata(seriesPath, false);
long count = timeseriesMetadata.getStatistics().getCount();
- costTime += (System.nanoTime() - startTime);
- System.out.println(count);
}
}
- System.out.println(
- "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms");
- System.out.println("Index area cost time: " + costTime / 1000_000 + "ms");
+ long totalTime = (System.nanoTime() - totalStartTime) / 1000_000;
+ System.out.println("Total raw read cost time: " + totalTime + "ms");
+ System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms");
}
}
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java
index 8681849..b9b52bf 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileAggregationV2.java
@@ -24,30 +24,38 @@ import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import java.io.IOException;
public class TsFileAggregationV2 {
- private static final String DEVICE1 = "device_";
- public static int chunkNum;
- public static int deviceNum = 1;
- public static int sensorNum = 1;
- public static int fileNum = 1;
+ private static final String DEVICE1 = "device_1";
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int fileNum;
public static void main(String[] args) throws IOException {
- long costTime = 0L;
Options opts = new Options();
- // Option chunkNumOption =
- // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c");
- // opts.addOption(chunkNumOption);
+ Option deviceNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d");
+ opts.addOption(deviceNumOption);
+ Option sensorNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m");
+ opts.addOption(sensorNumOption);
+ Option fileNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f");
+ opts.addOption(fileNumOption);
BasicParser parser = new BasicParser();
CommandLine cl;
try {
cl = parser.parse(opts, args);
- // chunkNum = Integer.parseInt(cl.getOptionValue("c"));
+ deviceNum = Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = Integer.parseInt(cl.getOptionValue("f"));
} catch (Exception e) {
e.printStackTrace();
}
@@ -56,22 +64,23 @@ public class TsFileAggregationV2 {
for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
// file path
String path =
- "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/"
+ "/data/szs/data/data/sequence/root.sg/0/"
+ deviceNum
- + "/test0.tsfile";
+ + "."
+ + sensorNum
+ + "/test"
+ + fileIndex
+ + ".tsfile";
// aggregation query
- try (TsFileSequenceReader reader = new TsFileSequenceReader(path)) {
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(path, false)) {
Path seriesPath = new Path(DEVICE1, "sensor_1");
- long startTime = System.nanoTime();
TimeseriesMetadata timeseriesMetadata = reader.readTimeseriesMetadataV4(seriesPath, false);
long count = timeseriesMetadata.getStatistics().getCount();
- costTime += (System.nanoTime() - startTime);
- System.out.println(count);
}
}
- System.out.println(
- "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms");
- System.out.println("Index area cost time: " + costTime / 1000_000 + "ms");
+ long totalTime = (System.nanoTime() - totalStartTime) / 1000_000;
+ System.out.println("Total raw read cost time: " + totalTime + "ms");
+ System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms");
}
}
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java
index 11d54b2..b3d1646 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawRead.java
@@ -26,6 +26,8 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import java.io.IOException;
@@ -33,24 +35,30 @@ import java.util.ArrayList;
public class TsFileRawRead {
- private static final String DEVICE1 = "device_";
- public static int chunkNum;
- public static int deviceNum = 1;
- public static int sensorNum = 1;
- public static int fileNum = 1;
+ private static final String DEVICE1 = "device_1";
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int fileNum;
public static void main(String[] args) throws IOException {
- long costTime = 0L;
Options opts = new Options();
- // Option chunkNumOption =
- // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c");
- // opts.addOption(chunkNumOption);
+ Option deviceNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d");
+ opts.addOption(deviceNumOption);
+ Option sensorNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m");
+ opts.addOption(sensorNumOption);
+ Option fileNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f");
+ opts.addOption(fileNumOption);
BasicParser parser = new BasicParser();
CommandLine cl;
try {
cl = parser.parse(opts, args);
- // chunkNum = Integer.parseInt(cl.getOptionValue("c"));
+ deviceNum = Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = Integer.parseInt(cl.getOptionValue("f"));
} catch (Exception e) {
e.printStackTrace();
}
@@ -59,9 +67,13 @@ public class TsFileRawRead {
for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
// file path
String path =
- "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/"
+ "/data/szs/data/data/sequence/root.sg/1/"
+ deviceNum
- + "/test1.tsfile";
+ + "."
+ + sensorNum
+ + "/test"
+ + fileIndex
+ + ".tsfile";
// raw data query
try (TsFileSequenceReader reader = new TsFileSequenceReader(path);
@@ -72,17 +84,14 @@ public class TsFileRawRead {
QueryExpression queryExpression = QueryExpression.create(paths, null);
- long startTime = System.nanoTime();
QueryDataSet queryDataSet = readTsFile.query(queryExpression);
while (queryDataSet.hasNext()) {
- System.out.println(queryDataSet.next());
+ queryDataSet.next();
}
-
- costTime += (System.nanoTime() - startTime);
}
}
- System.out.println(
- "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms");
- System.out.println("Index area cost time: " + costTime / 1000_000 + "ms");
+ long totalTime = (System.nanoTime() - totalStartTime) / 1000_000;
+ System.out.println("Total raw read cost time: " + totalTime + "ms");
+ System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms");
}
}
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java
index 2f898dc..96b2d59 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileRawReadV2.java
@@ -26,6 +26,8 @@ import org.apache.iotdb.tsfile.read.query.dataset.QueryDataSet;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import java.io.IOException;
@@ -33,38 +35,48 @@ import java.util.ArrayList;
public class TsFileRawReadV2 {
- private static final String DEVICE1 = "device_";
- public static int chunkNum;
- public static int deviceNum = 1;
- public static int sensorNum = 1;
- public static int fileNum = 1;
+ private static final String DEVICE1 = "device_1";
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int fileNum;
public static void main(String[] args) throws IOException {
- long costTime = 0L;
- long totalStartTime = System.nanoTime();
Options opts = new Options();
- // Option chunkNumOption =
- // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c");
- // opts.addOption(chunkNumOption);
+ Option deviceNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d");
+ opts.addOption(deviceNumOption);
+ Option sensorNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m");
+ opts.addOption(sensorNumOption);
+ Option fileNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f");
+ opts.addOption(fileNumOption);
BasicParser parser = new BasicParser();
CommandLine cl;
try {
cl = parser.parse(opts, args);
- // chunkNum = Integer.parseInt(cl.getOptionValue("c"));
+ deviceNum = Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = Integer.parseInt(cl.getOptionValue("f"));
} catch (Exception e) {
e.printStackTrace();
}
+ long totalStartTime = System.nanoTime();
for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
// file path
String path =
- "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/"
+ "/data/szs/data/data/sequence/root.sg/0/"
+ deviceNum
- + "/test0.tsfile";
+ + "."
+ + sensorNum
+ + "/test"
+ + fileIndex
+ + ".tsfile";
// raw data query
- try (TsFileSequenceReader reader = new TsFileSequenceReader(path);
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(path, false);
ReadOnlyTsFile readTsFile = new ReadOnlyTsFile(reader)) {
ArrayList<Path> paths = new ArrayList<>();
@@ -72,17 +84,14 @@ public class TsFileRawReadV2 {
QueryExpression queryExpression = QueryExpression.create(paths, null);
- long startTime = System.nanoTime();
QueryDataSet queryDataSet = readTsFile.query(queryExpression);
while (queryDataSet.hasNext()) {
- System.out.println(queryDataSet.next());
+ queryDataSet.next();
}
-
- costTime += (System.nanoTime() - startTime);
}
}
- System.out.println(
- "Total raw read cost time: " + (System.nanoTime() - totalStartTime) / 1000_000 + "ms");
- System.out.println("Index area cost time: " + costTime / 1000_000 + "ms");
+ long totalTime = (System.nanoTime() - totalStartTime) / 1000_000;
+ System.out.println("Total raw read cost time: " + totalTime + "ms");
+ System.out.println("Average cost time: " + (double) totalTime / (double) fileNum + "ms");
}
}
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileSketchToolV2.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileSketchToolV2.java
deleted file mode 100644
index c36a085..0000000
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileSketchToolV2.java
+++ /dev/null
@@ -1,425 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.iotdb.tsfile.test1835;
-
-import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
-import org.apache.iotdb.tsfile.file.MetaMarker;
-import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
-import org.apache.iotdb.tsfile.file.header.PageHeader;
-import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
-import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.IChunkMetadata;
-import org.apache.iotdb.tsfile.file.metadata.MetadataIndexEntry;
-import org.apache.iotdb.tsfile.file.metadata.MetadataIndexNode;
-import org.apache.iotdb.tsfile.file.metadata.TimeseriesMetadata;
-import org.apache.iotdb.tsfile.file.metadata.TsFileMetadata;
-import org.apache.iotdb.tsfile.file.metadata.enums.MetadataIndexNodeType;
-import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
-import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
-import org.apache.iotdb.tsfile.read.common.Chunk;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.utils.BloomFilter;
-import org.apache.iotdb.tsfile.utils.Pair;
-
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.TreeMap;
-
-public class TsFileSketchToolV2 {
-
- private String filename;
- private String indexFileName;
- private PrintWriter pw;
- private TsFileSketchToolReader reader;
- private TsFileSketchToolReader indexReader;
- private String splitStr; // for split different part of TsFile
-
- public static void main(String[] args) throws IOException {
- String filename = "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/1/test0.tsfile";
- String outFile = "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/1/test0.txt";
- String indexFileName =
- "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/1/test0.tsfile.index";
-
- new TsFileSketchToolV2(filename, indexFileName, outFile).run();
- }
-
- /**
- * construct TsFileSketchTool
- *
- * @param filename input file path
- * @param indexFileName index file path
- * @param outFile output file path
- */
- public TsFileSketchToolV2(String filename, String indexFileName, String outFile) {
- try {
- this.filename = filename;
- this.indexFileName = indexFileName;
- pw = new PrintWriter(new FileWriter(outFile));
- reader = new TsFileSketchToolReader(filename);
- indexReader = new TsFileSketchToolReader(indexFileName);
- StringBuilder str1 = new StringBuilder();
- for (int i = 0; i < 21; i++) {
- str1.append("|");
- }
- splitStr = str1.toString();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- /** entry of tool */
- public void run() throws IOException {
- long length = FSFactoryProducer.getFSFactory().getFile(filename).length();
- printlnBoth(
- pw, "-------------------------------- TsFile Sketch --------------------------------");
- printlnBoth(pw, "file path: " + filename);
- printlnBoth(pw, "file length: " + length);
-
- // get metadata information
- TsFileMetadata tsFileMetaData = reader.readFileMetadataV2();
- List<ChunkGroupMetadata> allChunkGroupMetadata = new ArrayList<>();
- reader.selfCheck(null, allChunkGroupMetadata, false);
-
- // print file information
- printFileInfo();
-
- // print chunk
- printChunk(allChunkGroupMetadata);
-
- // metadata begins
- if (tsFileMetaData.getMetadataIndex().getChildren().isEmpty()) {
- printlnBoth(pw, String.format("%20s", reader.getFileMetadataPos() - 1) + "|\t[marker] 2");
- } else {
- printlnBoth(
- pw, String.format("%20s", reader.readFileMetadata().getMetaOffset()) + "|\t[marker] 2");
- }
- // get all timeseries index
- Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap =
- reader.getAllTimeseriesMetadataWithOffset();
-
- // print timeseries index
- printTimeseriesIndex(timeseriesMetadataMap);
-
- // print TsFile Metadata
- printTsFileMetadata(tsFileMetaData);
-
- printlnBoth(pw, String.format("%20s", length) + "|\tEND of TsFile");
- printlnBoth(
- pw,
- "---------------------------- IndexOfTimerseriesIndex Tree -----------------------------");
- // print index tree
- MetadataIndexNode metadataIndexNode = tsFileMetaData.getMetadataIndex();
- TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap = new TreeMap<>();
- List<String> treeOutputStringBuffer = new ArrayList<>();
- loadIndexTree(metadataIndexNode, metadataIndexNodeMap, treeOutputStringBuffer, 0);
-
- // print IndexOfTimerseriesIndex
- printIndexOfTimerseriesIndex(metadataIndexNodeMap);
-
- for (String str : treeOutputStringBuffer) {
- printlnBoth(pw, str);
- }
- printlnBoth(
- pw,
- "---------------------------------- TsFile Sketch End ----------------------------------");
- pw.close();
- }
-
- private void printTsFileMetadata(TsFileMetadata tsFileMetaData) {
- try {
- printlnBoth(pw, String.format("%20s", reader.getFileMetadataPos()) + "|\t[TsFileMetadata]");
- printlnBoth(
- pw, String.format("%20s", "") + "|\t\t[meta offset] " + tsFileMetaData.getMetaOffset());
- // bloom filter
- BloomFilter bloomFilter = tsFileMetaData.getBloomFilter();
- printlnBoth(
- pw,
- String.format("%20s", "")
- + "|\t\t[bloom filter bit vector byte array length] "
- + bloomFilter.serialize().length);
- printlnBoth(pw, String.format("%20s", "") + "|\t\t[bloom filter bit vector byte array] ");
- printlnBoth(
- pw,
- String.format("%20s", "")
- + "|\t\t[bloom filter number of bits] "
- + bloomFilter.getSize());
- printlnBoth(
- pw,
- String.format("%20s", "")
- + "|\t\t[bloom filter number of hash functions] "
- + bloomFilter.getHashFunctionSize());
-
- printlnBoth(
- pw,
- String.format("%20s", (reader.getFileMetadataPos() + reader.getFileMetadataSize()))
- + "|\t[TsFileMetadataSize] "
- + reader.getFileMetadataSize());
-
- printlnBoth(
- pw,
- String.format("%20s", reader.getFileMetadataPos() + reader.getFileMetadataSize() + 4)
- + "|\t[magic tail] "
- + reader.readTailMagic());
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private void printIndexOfTimerseriesIndex(TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap) {
- for (Map.Entry<Long, MetadataIndexNode> entry : metadataIndexNodeMap.entrySet()) {
- printlnBoth(
- pw,
- String.format("%20s", entry.getKey())
- + "|\t[IndexOfTimerseriesIndex Node] type="
- + entry.getValue().getNodeType());
- for (MetadataIndexEntry metadataIndexEntry : entry.getValue().getChildren()) {
- printlnBoth(
- pw,
- String.format("%20s", "")
- + "|\t\t<"
- + metadataIndexEntry.getName()
- + ", "
- + metadataIndexEntry.getOffset()
- + ">");
- }
- printlnBoth(
- pw,
- String.format("%20s", "") + "|\t\t<endOffset, " + entry.getValue().getEndOffset() + ">");
- }
- }
-
- private void printFileInfo() {
- try {
- printlnBoth(pw, "");
- printlnBoth(pw, String.format("%20s", "POSITION") + "|\tCONTENT");
- printlnBoth(pw, String.format("%20s", "--------") + " \t-------");
- printlnBoth(pw, String.format("%20d", 0) + "|\t[magic head] " + reader.readHeadMagic());
- printlnBoth(
- pw,
- String.format("%20d", TSFileConfig.MAGIC_STRING.getBytes().length)
- + "|\t[version number] "
- + reader.readVersionNumber());
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private void printChunk(List<ChunkGroupMetadata> allChunkGroupMetadata) {
- try {
- long nextChunkGroupHeaderPos =
- (long) TSFileConfig.MAGIC_STRING.getBytes().length + Byte.BYTES;
- // ChunkGroup begins
- for (ChunkGroupMetadata chunkGroupMetadata : allChunkGroupMetadata) {
- printlnBoth(
- pw,
- splitStr
- + "\t[Chunk Group] of "
- + chunkGroupMetadata.getDevice()
- + ", num of Chunks:"
- + chunkGroupMetadata.getChunkMetadataList().size());
- // chunkGroupHeader begins
- printlnBoth(pw, String.format("%20s", nextChunkGroupHeaderPos) + "|\t[Chunk Group Header]");
- ChunkGroupHeader chunkGroupHeader =
- reader.readChunkGroupHeader(nextChunkGroupHeaderPos, false);
- printlnBoth(pw, String.format("%20s", "") + "|\t\t[marker] 0");
- printlnBoth(
- pw, String.format("%20s", "") + "|\t\t[deviceID] " + chunkGroupHeader.getDeviceID());
- // chunk begins
- for (ChunkMetadata chunkMetadata : chunkGroupMetadata.getChunkMetadataList()) {
- Chunk chunk = reader.readMemChunk(chunkMetadata);
- printlnBoth(
- pw,
- String.format("%20d", chunkMetadata.getOffsetOfChunkHeader())
- + "|\t[Chunk] of "
- + chunkMetadata.getMeasurementUid()
- + ", numOfPoints:"
- + chunkMetadata.getNumOfPoints()
- + ", time range:["
- + chunkMetadata.getStartTime()
- + ","
- + chunkMetadata.getEndTime()
- + "], tsDataType:"
- + chunkMetadata.getDataType()
- + ", \n"
- + String.format("%20s", "")
- + " \t"
- + chunkMetadata.getStatistics());
- printlnBoth(
- pw,
- String.format("%20s", "")
- + "|\t\t[chunk header] "
- + "marker="
- + chunk.getHeader().getChunkType()
- + ", measurementId="
- + chunk.getHeader().getMeasurementID()
- + ", dataSize="
- + chunk.getHeader().getDataSize()
- + ", serializedSize="
- + chunk.getHeader().getSerializedSize());
-
- printlnBoth(pw, String.format("%20s", "") + "|\t\t[chunk] " + chunk.getData());
- PageHeader pageHeader;
- if (((byte) (chunk.getHeader().getChunkType() & 0x3F))
- == MetaMarker.ONLY_ONE_PAGE_CHUNK_HEADER) {
- pageHeader = PageHeader.deserializeFrom(chunk.getData(), chunkMetadata.getStatistics());
- } else {
- pageHeader =
- PageHeader.deserializeFrom(chunk.getData(), chunk.getHeader().getDataType());
- }
- printlnBoth(
- pw,
- String.format("%20s", "")
- + "|\t\t[page] "
- + " CompressedSize:"
- + pageHeader.getCompressedSize()
- + ", UncompressedSize:"
- + pageHeader.getUncompressedSize());
- nextChunkGroupHeaderPos =
- chunkMetadata.getOffsetOfChunkHeader()
- + chunk.getHeader().getSerializedSize()
- + chunk.getHeader().getDataSize();
- }
- reader.position(nextChunkGroupHeaderPos);
- byte marker = reader.readMarker();
- switch (marker) {
- case MetaMarker.CHUNK_GROUP_HEADER:
- // do nothing
- break;
- case MetaMarker.OPERATION_INDEX_RANGE:
- // skip the PlanIndex
- nextChunkGroupHeaderPos += 16;
- break;
- }
-
- printlnBoth(
- pw, splitStr + "\t[Chunk Group] of " + chunkGroupMetadata.getDevice() + " ends");
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- private void printTimeseriesIndex(
- Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap) {
- try {
- for (Map.Entry<Long, Pair<Path, TimeseriesMetadata>> entry :
- timeseriesMetadataMap.entrySet()) {
- printlnBoth(
- pw,
- String.format("%20s", entry.getKey())
- + "|\t[TimeseriesIndex] of "
- + entry.getValue().left
- + ", tsDataType:"
- + entry.getValue().right.getTSDataType());
- for (IChunkMetadata chunkMetadata :
- reader.getChunkMetadataListV3(entry.getValue().left, false)) {
- printlnBoth(
- pw,
- String.format("%20s", "")
- + "|\t\t[ChunkIndex] "
- + chunkMetadata.getMeasurementUid()
- + ", offset="
- + chunkMetadata.getOffsetOfChunkHeader());
- }
- printlnBoth(
- pw,
- String.format("%20s", "") + "|\t\t[" + entry.getValue().right.getStatistics() + "] ");
- }
- printlnBoth(pw, splitStr);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- /**
- * load by dfs, and sort by TreeMap
- *
- * @param metadataIndexNode current node
- * @param metadataIndexNodeMap result map, key is offset
- * @param treeOutputStringBuffer result list, string is index tree
- * @param deep current deep
- */
- private void loadIndexTree(
- MetadataIndexNode metadataIndexNode,
- TreeMap<Long, MetadataIndexNode> metadataIndexNodeMap,
- List<String> treeOutputStringBuffer,
- int deep)
- throws IOException {
- StringBuilder tableWriter = new StringBuilder("\t");
- for (int i = 0; i < deep; i++) {
- tableWriter.append("\t\t");
- }
- treeOutputStringBuffer.add(
- tableWriter.toString() + "[MetadataIndex:" + metadataIndexNode.getNodeType() + "]");
- for (int i = 0; i < metadataIndexNode.getChildren().size(); i++) {
- MetadataIndexEntry metadataIndexEntry = metadataIndexNode.getChildren().get(i);
-
- treeOutputStringBuffer.add(
- tableWriter.toString()
- + "└──────["
- + metadataIndexEntry.getName()
- + ","
- + metadataIndexEntry.getOffset()
- + "]");
- if (!metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_MEASUREMENT)) {
- long endOffset = metadataIndexNode.getEndOffset();
- if (i != metadataIndexNode.getChildren().size() - 1) {
- endOffset = metadataIndexNode.getChildren().get(i + 1).getOffset();
- }
- MetadataIndexNode subNode =
- indexReader.getMetadataIndexNode(metadataIndexEntry.getOffset(), endOffset);
- metadataIndexNodeMap.put(metadataIndexEntry.getOffset(), subNode);
- loadIndexTree(subNode, metadataIndexNodeMap, treeOutputStringBuffer, deep + 1);
- }
- }
- }
-
- private void printlnBoth(PrintWriter pw, String str) {
- System.out.println(str);
- pw.println(str);
- }
-
- private class TsFileSketchToolReader extends TsFileSequenceReader {
- public TsFileSketchToolReader(String file) throws IOException {
- super(file);
- }
-
- public Map<Long, Pair<Path, TimeseriesMetadata>> getAllTimeseriesMetadataWithOffset()
- throws IOException {
- Map<Long, Pair<Path, TimeseriesMetadata>> timeseriesMetadataMap = new TreeMap<>();
-
- // FIXME
- ByteBuffer buffer = readData(0, 0);
- while (buffer.hasRemaining()) {
- int bufferPos = buffer.position();
- TimeseriesMetadata timeseriesMetaData = TimeseriesMetadata.deserializeFrom(buffer, false);
- timeseriesMetadataMap.put(
- reader.position() + bufferPos,
- new Pair<>(new Path("d1", timeseriesMetaData.getMeasurementId()), timeseriesMetaData));
- }
- return timeseriesMetadataMap;
- }
- }
-}
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java
index aa087b4..d6c64d5 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java
@@ -32,6 +32,8 @@ import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import java.io.File;
@@ -42,21 +44,30 @@ import java.util.Random;
* addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException
*/
public class TsFileWrite {
- public static int deviceNum = 1;
- public static int sensorNum = 1;
- public static int fileNum = 1;
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int fileNum;
+ public static int pointNum = 100;
public static void main(String[] args) {
Options opts = new Options();
- // Option chunkNumOption =
- // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c");
- // opts.addOption(chunkNumOption);
+ Option deviceNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d");
+ opts.addOption(deviceNumOption);
+ Option sensorNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m");
+ opts.addOption(sensorNumOption);
+ Option fileNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f");
+ opts.addOption(fileNumOption);
BasicParser parser = new BasicParser();
CommandLine cl;
try {
cl = parser.parse(opts, args);
- // chunkNum = Integer.parseInt(cl.getOptionValue("c"));
+ deviceNum = Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = Integer.parseInt(cl.getOptionValue("f"));
} catch (Exception e) {
e.printStackTrace();
}
@@ -64,10 +75,12 @@ public class TsFileWrite {
for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
try {
String path =
- "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/"
+ "/data/szs/data/data/sequence/root.sg/0/"
+ deviceNum
+ + "."
+ + sensorNum
+ "/test"
- + 0
+ + fileIndex
+ ".tsfile";
File f = FSFactoryProducer.getFSFactory().getFile(path);
if (f.exists()) {
@@ -76,23 +89,29 @@ public class TsFileWrite {
try {
TsFileWriter tsFileWriter = new TsFileWriter(f);
- // 1000 timeseries
- for (int i = 1; i <= 1000; i++) {
- tsFileWriter.registerTimeseries(
- new Path(Constant.DEVICE_PREFIX, Constant.SENSOR_ + i),
- new UnaryMeasurementSchema(Constant.SENSOR_ + i, TSDataType.INT64, TSEncoding.RLE));
+ for (int i = 1; i <= deviceNum; i++) {
+ for (int j = 1; j <= sensorNum; j++) {
+ Path path1 = new Path(Constant.DEVICE_PREFIX + i, Constant.SENSOR_ + j);
+ tsFileWriter.registerTimeseries(
+ path1,
+ new UnaryMeasurementSchema(
+ Constant.SENSOR_ + j, TSDataType.INT64, TSEncoding.RLE));
+ }
}
// construct TSRecord
- for (int i = 1; i <= 100; i++) {
- TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX);
- for (int t = 1; t <= 1000; t++) {
- DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong());
- tsRecord.addTuple(dPoint1);
- }
- // write TSRecord
- tsFileWriter.write(tsRecord);
- if (i % 100 == 0) {
- tsFileWriter.flushAllChunkGroups();
+ for (int j = 1; j <= deviceNum; j++) {
+ for (int i = 1; i <= pointNum; i++) {
+ TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + j);
+ for (int t = 1; t <= sensorNum; t++) {
+ DataPoint dPoint1 =
+ new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong());
+ tsRecord.addTuple(dPoint1);
+ }
+ // write TSRecord
+ tsFileWriter.write(tsRecord);
+ if (i % 100 == 0) {
+ tsFileWriter.flushAllChunkGroups();
+ }
}
}
tsFileWriter.close();
diff --git a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWriteV2.java
similarity index 59%
copy from example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java
copy to example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWriteV2.java
index aa087b4..ffc8cc0 100644
--- a/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWrite.java
+++ b/example/tsfile/src/main/java/org/apache/iotdb/tsfile/test1835/TsFileWriteV2.java
@@ -32,6 +32,8 @@ import org.apache.iotdb.tsfile.write.schema.UnaryMeasurementSchema;
import org.apache.commons.cli.BasicParser;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import java.io.File;
@@ -41,22 +43,31 @@ import java.util.Random;
* An example of writing data with TSRecord to TsFile It uses the interface: public void
* addMeasurement(MeasurementSchema measurementSchema) throws WriteProcessException
*/
-public class TsFileWrite {
- public static int deviceNum = 1;
- public static int sensorNum = 1;
- public static int fileNum = 1;
+public class TsFileWriteV2 {
+ public static int deviceNum;
+ public static int sensorNum;
+ public static int fileNum;
+ public static int pointNum = 100;
public static void main(String[] args) {
Options opts = new Options();
- // Option chunkNumOption =
- // OptionBuilder.withArgName("args").withLongOpt("chunkNum").hasArg().create("c");
- // opts.addOption(chunkNumOption);
+ Option deviceNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("deviceNum").hasArg().create("d");
+ opts.addOption(deviceNumOption);
+ Option sensorNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("sensorNum").hasArg().create("m");
+ opts.addOption(sensorNumOption);
+ Option fileNumOption =
+ OptionBuilder.withArgName("args").withLongOpt("fileNum").hasArg().create("f");
+ opts.addOption(fileNumOption);
BasicParser parser = new BasicParser();
CommandLine cl;
try {
cl = parser.parse(opts, args);
- // chunkNum = Integer.parseInt(cl.getOptionValue("c"));
+ deviceNum = 1000; // Integer.parseInt(cl.getOptionValue("d"));
+ sensorNum = 10; // Integer.parseInt(cl.getOptionValue("m"));
+ fileNum = 1; // Integer.parseInt(cl.getOptionValue("f"));
} catch (Exception e) {
e.printStackTrace();
}
@@ -64,10 +75,12 @@ public class TsFileWrite {
for (int fileIndex = 0; fileIndex < fileNum; fileIndex++) {
try {
String path =
- "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/1/"
+ "/Users/samperson1997/git/iotdb/data/data/sequence/root.sg/0/"
+ deviceNum
+ + "."
+ + sensorNum
+ "/test"
- + 0
+ + fileIndex
+ ".tsfile";
File f = FSFactoryProducer.getFSFactory().getFile(path);
if (f.exists()) {
@@ -76,23 +89,29 @@ public class TsFileWrite {
try {
TsFileWriter tsFileWriter = new TsFileWriter(f);
- // 1000 timeseries
- for (int i = 1; i <= 1000; i++) {
- tsFileWriter.registerTimeseries(
- new Path(Constant.DEVICE_PREFIX, Constant.SENSOR_ + i),
- new UnaryMeasurementSchema(Constant.SENSOR_ + i, TSDataType.INT64, TSEncoding.RLE));
+ for (int i = 1; i <= deviceNum; i++) {
+ for (int j = 1; j <= sensorNum; j++) {
+ Path path1 = new Path(Constant.DEVICE_PREFIX + i, Constant.SENSOR_ + j);
+ tsFileWriter.registerTimeseries(
+ path1,
+ new UnaryMeasurementSchema(
+ Constant.SENSOR_ + j, TSDataType.INT64, TSEncoding.RLE));
+ }
}
// construct TSRecord
- for (int i = 1; i <= 100; i++) {
- TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX);
- for (int t = 1; t <= 1000; t++) {
- DataPoint dPoint1 = new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong());
- tsRecord.addTuple(dPoint1);
- }
- // write TSRecord
- tsFileWriter.write(tsRecord);
- if (i % 100 == 0) {
- tsFileWriter.flushAllChunkGroups();
+ for (int j = 1; j <= deviceNum; j++) {
+ for (int i = 1; i <= pointNum; i++) {
+ TSRecord tsRecord = new TSRecord(i, Constant.DEVICE_PREFIX + j);
+ for (int t = 1; t <= sensorNum; t++) {
+ DataPoint dPoint1 =
+ new LongDataPoint(Constant.SENSOR_ + t, new Random().nextLong());
+ tsRecord.addTuple(dPoint1);
+ }
+ // write TSRecord
+ tsFileWriter.write(tsRecord);
+ if (i % 100 == 0) {
+ tsFileWriter.flushAllChunkGroups();
+ }
}
}
tsFileWriter.close();
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index bf5856d..712a812 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -103,6 +103,8 @@ public class TsFileSequenceReader implements AutoCloseable {
private long minPlanIndex = Long.MAX_VALUE;
private long maxPlanIndex = Long.MIN_VALUE;
+ private long startTime;
+
/**
* Create a file reader of the given file. The reader will read the tail of the file to get the
* file metadata size.Then the reader will skip the first
@@ -131,14 +133,14 @@ public class TsFileSequenceReader implements AutoCloseable {
if (FSFactoryProducer.getFSFactory().getFile(file + ".index").exists()) {
metadataIndexInput = FSFactoryProducer.getFileInputFactory().getTsFileInput(file + ".index");
}
- try {
- if (loadMetadataSize) {
- loadMetadataSize();
- }
- } catch (Throwable e) {
- tsFileInput.close();
- throw e;
- }
+ // try {
+ // if (loadMetadataSize) {
+ // loadMetadataSize();
+ // }
+ // } catch (Throwable e) {
+ // tsFileInput.close();
+ // throw e;
+ // }
}
// used in merge resource
@@ -195,19 +197,19 @@ public class TsFileSequenceReader implements AutoCloseable {
public void loadMetadataSize() throws IOException {
ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES);
- if (readTailMagic().equals(TSFileConfig.MAGIC_STRING)) {
- tsFileInput.read(
- metadataSize,
- tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES);
- metadataSize.flip();
- // read file metadata size and position
- fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize);
- fileMetadataPos =
- tsFileInput.size()
- - TSFileConfig.MAGIC_STRING.getBytes().length
- - Integer.BYTES
- - fileMetadataSize;
- }
+ // if (readTailMagic().equals(TSFileConfig.MAGIC_STRING)) {
+ tsFileInput.read(
+ metadataSize,
+ tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES);
+ metadataSize.flip();
+ // read file metadata size and position
+ fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize);
+ fileMetadataPos =
+ tsFileInput.size()
+ - TSFileConfig.MAGIC_STRING.getBytes().length
+ - Integer.BYTES
+ - fileMetadataSize;
+ // }
}
public long getFileMetadataPos() {
@@ -265,8 +267,25 @@ public class TsFileSequenceReader implements AutoCloseable {
public TsFileMetadata readFileMetadata() throws IOException {
try {
if (tsFileMetaData == null) {
+ long start = System.nanoTime();
+ ByteBuffer metadataSize = ByteBuffer.allocate(Integer.BYTES);
+ tsFileInput.read(
+ metadataSize,
+ tsFileInput.size() - TSFileConfig.MAGIC_STRING.getBytes().length - Integer.BYTES);
+ metadataSize.flip();
+
+ // read file metadata size and position
+ fileMetadataSize = ReadWriteIOUtils.readInt(metadataSize);
+
+ fileMetadataPos =
+ tsFileInput.size()
+ - TSFileConfig.MAGIC_STRING.getBytes().length
+ - Integer.BYTES
+ - fileMetadataSize;
+ resourceLogger.debug("ReadFileMetadata " + (System.nanoTime() - start) / 1000_000L + " ms");
tsFileMetaData =
TsFileMetadata.deserializeFrom(readData(fileMetadataPos, fileMetadataSize));
+ resourceLogger.debug("ReadFileMetadata " + (System.nanoTime() - start) / 1000_000L + " ms");
}
} catch (BufferOverflowException e) {
logger.error("Something error happened while reading file metadata of file {}", file);
@@ -278,18 +297,23 @@ public class TsFileSequenceReader implements AutoCloseable {
public TsFileMetadata readFileMetadataV2() throws IOException {
try {
if (tsFileMetaData == null) {
- long totalSize = metadataIndexInput.size();
+ long start = System.nanoTime();
ByteBuffer rootNodeOffsetBuffer = ByteBuffer.allocate(Long.BYTES);
- metadataIndexInput.read(rootNodeOffsetBuffer, totalSize - Long.BYTES);
+ metadataIndexInput.read(rootNodeOffsetBuffer, metadataIndexInput.size() - Long.BYTES);
rootNodeOffsetBuffer.flip();
long rootNodeOffset = ReadWriteIOUtils.readLong(rootNodeOffsetBuffer);
+ resourceLogger.debug(
+ "ReadFileMetadataV2 " + (System.nanoTime() - start) / 1000_000L + " ms");
+
tsFileMetaData =
TsFileMetadataV2.deserializeFrom(
readData(
rootNodeOffset,
FSFactoryProducer.getFSFactory().getFile(this.file + ".index").length(),
metadataIndexInput));
+ resourceLogger.debug(
+ "ReadFileMetadataV2 " + (System.nanoTime() - start) / 1000_000L + " ms");
}
} catch (BufferOverflowException e) {
logger.error("Something error happened while reading file metadata of file {}", file);
@@ -443,33 +467,12 @@ public class TsFileSequenceReader implements AutoCloseable {
return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
}
- public TimeseriesMetadata readTimeseriesMetadataV3(Path path, boolean ignoreNotExists)
- throws IOException {
- readFileMetadataV2();
-
- List<TimeseriesMetadata> timeseriesMetadataList = new ArrayList<>();
- ByteBuffer buffer = readData(position(), fileMetadataPos);
- while (buffer.hasRemaining()) {
- try {
- timeseriesMetadataList.add(TimeseriesMetadata.deserializeFrom(buffer, true));
- } catch (BufferOverflowException e) {
- logger.error(
- "Something error happened while deserializing TimeseriesMetadata of file {}", file);
- throw e;
- }
- }
- // return null if path does not exist in the TsFile
- int searchResult =
- binarySearchInTimeseriesMetadataList(timeseriesMetadataList, path.getMeasurement());
- return searchResult >= 0 ? timeseriesMetadataList.get(searchResult) : null;
- }
-
public TimeseriesMetadata readTimeseriesMetadataV4(Path path, boolean ignoreNotExists)
throws IOException {
readFileMetadataV2();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair =
- getMetadataAndEndOffset(deviceMetadataIndexNode, path.getDevice(), true, true);
+ getMetadataAndEndOffsetV2(deviceMetadataIndexNode, path.getDevice(), true, true);
if (metadataIndexPair == null) {
if (ignoreNotExists) {
return null;
@@ -487,7 +490,7 @@ public class TsFileSequenceReader implements AutoCloseable {
throw e;
}
metadataIndexPair =
- getMetadataAndEndOffset(metadataIndexNode, path.getMeasurement(), false, false);
+ getMetadataAndEndOffsetV2(metadataIndexNode, path.getMeasurement(), false, false);
}
if (metadataIndexPair == null) {
return null;
@@ -596,6 +599,7 @@ public class TsFileSequenceReader implements AutoCloseable {
public List<TimeseriesMetadata> readTimeseriesMetadata(String device, Set<String> measurements)
throws IOException {
+ long start = System.nanoTime();
readFileMetadata();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair =
@@ -649,6 +653,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
if (measurementsHadFound.size() == measurements.size()) {
+ resourceLogger.debug(
+ "ReadTimeseriesMetadata " + (System.nanoTime() - start) / 1000_000L + " ms");
return resultTimeseriesMetadataList;
}
}
@@ -721,10 +727,12 @@ public class TsFileSequenceReader implements AutoCloseable {
public List<TimeseriesMetadata> readTimeseriesMetadataV3(String device, Set<String> measurements)
throws IOException {
+ long start = System.nanoTime();
+
readFileMetadataV2();
MetadataIndexNode deviceMetadataIndexNode = tsFileMetaData.getMetadataIndex();
Pair<MetadataIndexEntry, Long> metadataIndexPair =
- getMetadataAndEndOffset(deviceMetadataIndexNode, device, true, false);
+ getMetadataAndEndOffsetV2(deviceMetadataIndexNode, device, true, false);
if (metadataIndexPair == null) {
return Collections.emptyList();
}
@@ -748,7 +756,7 @@ public class TsFileSequenceReader implements AutoCloseable {
throw e;
}
measurementMetadataIndexPair =
- getMetadataAndEndOffset(metadataIndexNode, measurementList.get(i), false, false);
+ getMetadataAndEndOffsetV2(metadataIndexNode, measurementList.get(i), false, false);
}
if (measurementMetadataIndexPair == null) {
return Collections.emptyList();
@@ -775,6 +783,8 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
if (measurementsHadFound.size() == measurements.size()) {
+ resourceLogger.debug(
+ "ReadTimeseriesMetadataV3 " + (System.nanoTime() - start) / 1000_000L + " ms");
return resultTimeseriesMetadataList;
}
}
@@ -782,19 +792,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return resultTimeseriesMetadataList;
}
- public MetadataIndexNode readMetadataIndex() throws IOException {
- long totalSize = metadataIndexInput.size();
- ByteBuffer lastNodeSizeBuffer = ByteBuffer.allocate(Integer.BYTES);
- metadataIndexInput.read(lastNodeSizeBuffer, totalSize - Integer.BYTES);
- lastNodeSizeBuffer.flip();
-
- int lastNodeSize = ReadWriteIOUtils.readInt(lastNodeSizeBuffer);
- ByteBuffer lastNode = ByteBuffer.allocate(lastNodeSize);
- metadataIndexInput.read(lastNode, totalSize - lastNodeSize - Integer.BYTES);
- lastNode.flip();
- return MetadataIndexNode.deserializeFrom(lastNode);
- }
-
protected int binarySearchInTimeseriesMetadataList(
List<TimeseriesMetadata> timeseriesMetadataList, String key) {
int low = 0;
@@ -915,7 +912,7 @@ public class TsFileSequenceReader implements AutoCloseable {
.collect(Collectors.toList()));
} else {
// keep traversing
- deviceList.addAll(getAllDevices(node));
+ deviceList.addAll(getAllDevicesV2(node));
}
}
return deviceList;
@@ -1127,6 +1124,31 @@ public class TsFileSequenceReader implements AutoCloseable {
}
}
+ protected Pair<MetadataIndexEntry, Long> getMetadataAndEndOffsetV2(
+ MetadataIndexNode metadataIndex, String name, boolean isDeviceLevel, boolean exactSearch)
+ throws IOException {
+ try {
+ // When searching for a device node, return when it is not INTERNAL_DEVICE
+ // When searching for a measurement node, return when it is not INTERNAL_MEASUREMENT
+ if ((isDeviceLevel
+ && !metadataIndex.getNodeType().equals(MetadataIndexNodeType.INTERNAL_DEVICE))
+ || (!isDeviceLevel
+ && !metadataIndex.getNodeType().equals(MetadataIndexNodeType.INTERNAL_MEASUREMENT))) {
+ return metadataIndex.getChildIndexEntry(name, exactSearch);
+ } else {
+ Pair<MetadataIndexEntry, Long> childIndexEntry =
+ metadataIndex.getChildIndexEntry(name, false);
+ ByteBuffer buffer =
+ readData(childIndexEntry.left.getOffset(), childIndexEntry.right, metadataIndexInput);
+ return getMetadataAndEndOffsetV2(
+ MetadataIndexNode.deserializeFrom(buffer), name, isDeviceLevel, exactSearch);
+ }
+ } catch (BufferOverflowException e) {
+ logger.error("Something error happened while deserializing MetadataIndex of file {}", file);
+ throw e;
+ }
+ }
+
/**
* read data from current position of the input, and deserialize it to a CHUNK_GROUP_FOOTER. <br>
* This method is not threadsafe.
@@ -1590,15 +1612,6 @@ public class TsFileSequenceReader implements AutoCloseable {
return chunkMetadataList;
}
- public List<ChunkMetadata> getChunkMetadataListV3(Path path, boolean ignoreNotExists)
- throws IOException {
- TimeseriesMetadata timeseriesMetaData = readTimeseriesMetadataV3(path, ignoreNotExists);
-
- List<ChunkMetadata> chunkMetadataList = readChunkMetaDataList(timeseriesMetaData);
- chunkMetadataList.sort(Comparator.comparingLong(IChunkMetadata::getStartTime));
- return chunkMetadataList;
- }
-
public List<ChunkMetadata> getChunkMetadataList(Path path) throws IOException {
return getChunkMetadataList(path, false);
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
index b3b43a2..cd547ca 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/query/executor/TsFileExecutor.java
@@ -67,8 +67,7 @@ public class TsFileExecutor implements QueryExecutor {
queryExpression.setSelectSeries(filteredSeriesPath);
}
- // metadataQuerier.loadChunkMetaDatas(queryExpression.getSelectedSeries());
- // metadataQuerier.loadChunkMetaDatasV2(queryExpression.getSelectedSeries());
+ // metadataQuerier.loadChunkMetaDatas(queryExpression.getSelectedSeries());
metadataQuerier.loadChunkMetaDatasV3(queryExpression.getSelectedSeries());
if (queryExpression.hasQueryFilter()) {
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
index 4dd528d..625c38c 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/TsFileWriter.java
@@ -356,7 +356,7 @@ public class TsFileWriter implements AutoCloseable {
// }
//
// public void closeV2() throws IOException {
- LOG.info("start close file");
+ LOG.info("start close file IN NEW WAY");
flushAllChunkGroups();
fileWriter.endFileV3();
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
index bba12dc..ac24f39 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/writer/TsFileIOWriter.java
@@ -331,86 +331,93 @@ public class TsFileIOWriter {
}
canWrite = false;
}
-
- public void endFileV2() throws IOException {
- long metaOffset = out.getPosition();
-
- // serialize the SEPARATOR of MetaData
- ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
-
- // group ChunkMetadata by series
- // only contains ordinary path and time column of vector series
- Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>();
-
- // time column -> ChunkMetadataList TreeMap of value columns in vector
- Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>();
-
- for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
- List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList();
- int idx = 0;
- while (idx < chunkMetadatas.size()) {
- IChunkMetadata chunkMetadata = chunkMetadatas.get(idx);
- if (chunkMetadata.getMask() == 0) {
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- } else if (chunkMetadata.isTimeColumn()) {
- // time column of a vector series
- Path series = new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMap.computeIfAbsent(series, k -> new ArrayList<>()).add(chunkMetadata);
- idx++;
- Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector =
- vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>());
-
- // value columns of a vector series
- while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) {
- chunkMetadata = chunkMetadatas.get(idx);
- Path vectorSeries =
- new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
- chunkMetadataListMapInVector
- .computeIfAbsent(vectorSeries, k -> new ArrayList<>())
- .add(chunkMetadata);
- idx++;
- }
- }
- }
- }
-
- MetadataIndexNode metadataIndex = flushMetadataIndexV2(chunkMetadataListMap, vectorToPathsMap);
- TsFileMetadata tsFileMetaData = new TsFileMetadata();
- tsFileMetaData.setMetadataIndex(metadataIndex);
- tsFileMetaData.setMetaOffset(metaOffset);
-
- long footerIndex = out.getPosition();
- if (logger.isDebugEnabled()) {
- logger.debug("start to flush the footer,file pos:{}", footerIndex);
- }
-
- // write TsFileMetaData
- int size = tsFileMetaData.serializeTo(out.wrapAsStream());
- if (logger.isDebugEnabled()) {
- logger.debug("finish flushing the footer {}, file pos:{}", tsFileMetaData, out.getPosition());
- }
-
- // write bloom filter
- size += tsFileMetaData.serializeBloomFilter(out.wrapAsStream(), chunkMetadataListMap.keySet());
- if (logger.isDebugEnabled()) {
- logger.debug("finish flushing the bloom filter file pos:{}", out.getPosition());
- }
-
- // write TsFileMetaData size
- ReadWriteIOUtils.write(size, out.wrapAsStream()); // write the size of the file metadata.
-
- // write magic string
- out.write(MAGIC_STRING_BYTES);
-
- // close file
- out.close();
- if (resourceLogger.isDebugEnabled() && file != null) {
- resourceLogger.debug("{} writer is closed.", file.getName());
- }
- canWrite = false;
- }
+ //
+ // public void endFileV2() throws IOException {
+ // long metaOffset = out.getPosition();
+ //
+ // // serialize the SEPARATOR of MetaData
+ // ReadWriteIOUtils.write(MetaMarker.SEPARATOR, out.wrapAsStream());
+ //
+ // // group ChunkMetadata by series
+ // // only contains ordinary path and time column of vector series
+ // Map<Path, List<IChunkMetadata>> chunkMetadataListMap = new TreeMap<>();
+ //
+ // // time column -> ChunkMetadataList TreeMap of value columns in vector
+ // Map<Path, Map<Path, List<IChunkMetadata>>> vectorToPathsMap = new HashMap<>();
+ //
+ // for (ChunkGroupMetadata chunkGroupMetadata : chunkGroupMetadataList) {
+ // List<ChunkMetadata> chunkMetadatas = chunkGroupMetadata.getChunkMetadataList();
+ // int idx = 0;
+ // while (idx < chunkMetadatas.size()) {
+ // IChunkMetadata chunkMetadata = chunkMetadatas.get(idx);
+ // if (chunkMetadata.getMask() == 0) {
+ // Path series = new Path(chunkGroupMetadata.getDevice(),
+ // chunkMetadata.getMeasurementUid());
+ // chunkMetadataListMap.computeIfAbsent(series, k -> new
+ // ArrayList<>()).add(chunkMetadata);
+ // idx++;
+ // } else if (chunkMetadata.isTimeColumn()) {
+ // // time column of a vector series
+ // Path series = new Path(chunkGroupMetadata.getDevice(),
+ // chunkMetadata.getMeasurementUid());
+ // chunkMetadataListMap.computeIfAbsent(series, k -> new
+ // ArrayList<>()).add(chunkMetadata);
+ // idx++;
+ // Map<Path, List<IChunkMetadata>> chunkMetadataListMapInVector =
+ // vectorToPathsMap.computeIfAbsent(series, key -> new TreeMap<>());
+ //
+ // // value columns of a vector series
+ // while (idx < chunkMetadatas.size() && chunkMetadatas.get(idx).isValueColumn()) {
+ // chunkMetadata = chunkMetadatas.get(idx);
+ // Path vectorSeries =
+ // new Path(chunkGroupMetadata.getDevice(), chunkMetadata.getMeasurementUid());
+ // chunkMetadataListMapInVector
+ // .computeIfAbsent(vectorSeries, k -> new ArrayList<>())
+ // .add(chunkMetadata);
+ // idx++;
+ // }
+ // }
+ // }
+ // }
+ //
+ // MetadataIndexNode metadataIndex = flushMetadataIndexV2(chunkMetadataListMap,
+ // vectorToPathsMap);
+ // TsFileMetadata tsFileMetaData = new TsFileMetadata();
+ // tsFileMetaData.setMetadataIndex(metadataIndex);
+ // tsFileMetaData.setMetaOffset(metaOffset);
+ //
+ // long footerIndex = out.getPosition();
+ // if (logger.isDebugEnabled()) {
+ // logger.debug("start to flush the footer,file pos:{}", footerIndex);
+ // }
+ //
+ // // write TsFileMetaData
+ // int size = tsFileMetaData.serializeTo(out.wrapAsStream());
+ // if (logger.isDebugEnabled()) {
+ // logger.debug("finish flushing the footer {}, file pos:{}", tsFileMetaData,
+ // out.getPosition());
+ // }
+ //
+ // // write bloom filter
+ // size += tsFileMetaData.serializeBloomFilter(out.wrapAsStream(),
+ // chunkMetadataListMap.keySet());
+ // if (logger.isDebugEnabled()) {
+ // logger.debug("finish flushing the bloom filter file pos:{}", out.getPosition());
+ // }
+ //
+ // // write TsFileMetaData size
+ // ReadWriteIOUtils.write(size, out.wrapAsStream()); // write the size of the file metadata.
+ //
+ // // write magic string
+ // out.write(MAGIC_STRING_BYTES);
+ //
+ // // close file
+ // out.close();
+ // if (resourceLogger.isDebugEnabled() && file != null) {
+ // resourceLogger.debug("{} writer is closed.", file.getName());
+ // }
+ // canWrite = false;
+ // }
public void endFileV3() throws IOException {
long metaOffset = out.getPosition();
@@ -497,7 +504,7 @@ public class TsFileIOWriter {
// close file
out.close();
if (resourceLogger.isDebugEnabled() && file != null) {
- resourceLogger.debug("{} writer is closed.", file.getName());
+ resourceLogger.debug("{} writer is closed IN NEW WAY.", file.getName());
}
canWrite = false;
}