You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2021/06/15 03:04:58 UTC
[iotdb] branch master updated: [IOTDB-1437]Fix the tsfile sketch
tool NPE (#3397)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6384f07 [IOTDB-1437]Fix the tsfile sketch tool NPE (#3397)
6384f07 is described below
commit 6384f075536835801a7588a6192c81db8d619900
Author: Houliang Qi <ne...@163.com>
AuthorDate: Tue Jun 15 11:04:01 2021 +0800
[IOTDB-1437]Fix the tsfile sketch tool NPE (#3397)
---
.../apache/iotdb/db/tools/TsFileSketchTool.java | 15 ++-
.../iotdb/db/tools/TsFileSketchToolTest.java | 133 +++++++++++++++++++++
.../theme/global-components/Contributor.vue | 2 +-
3 files changed, 147 insertions(+), 3 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
index c33d2ac..d36f736 100644
--- a/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
+++ b/server/src/main/java/org/apache/iotdb/db/tools/TsFileSketchTool.java
@@ -20,6 +20,7 @@
package org.apache.iotdb.db.tools;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.MetaMarker;
import org.apache.iotdb.tsfile.file.header.ChunkGroupHeader;
import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetadata;
import org.apache.iotdb.tsfile.file.metadata.ChunkMetadata;
@@ -122,8 +123,18 @@ public class TsFileSketchTool {
nextChunkGroupHeaderPos =
chunkMetadata.getOffsetOfChunkHeader()
+ chunk.getHeader().getSerializedSize()
- + chunk.getHeader().getDataSize()
- + 17; // skip the PlanIndex
+ + chunk.getHeader().getDataSize();
+ }
+ reader.position(nextChunkGroupHeaderPos);
+ byte marker = reader.readMarker();
+ switch (marker) {
+ case MetaMarker.CHUNK_GROUP_HEADER:
+ // do nothing
+ break;
+ case MetaMarker.OPERATION_INDEX_RANGE:
+ // skip the PlanIndex
+ nextChunkGroupHeaderPos += 16;
+ break;
}
printlnBoth(pw, str1 + "\t[Chunk Group] of " + chunkGroupMetadata.getDevice() + " ends");
diff --git a/server/src/test/java/org/apache/iotdb/db/tools/TsFileSketchToolTest.java b/server/src/test/java/org/apache/iotdb/db/tools/TsFileSketchToolTest.java
new file mode 100644
index 0000000..8a9a143
--- /dev/null
+++ b/server/src/test/java/org/apache/iotdb/db/tools/TsFileSketchToolTest.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.tools;
+
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
+import org.apache.iotdb.tsfile.read.common.Path;
+import org.apache.iotdb.tsfile.write.TsFileWriter;
+import org.apache.iotdb.tsfile.write.record.Tablet;
+import org.apache.iotdb.tsfile.write.schema.IMeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
+
+import org.apache.commons.io.FileUtils;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+public class TsFileSketchToolTest {
+ String path = "test.tsfile";
+ String sketchOut = "sketch.out";
+ String device = "root.device_0";
+
+ @Before
+ public void setUp() throws Exception {
+ try {
+ File f = FSFactoryProducer.getFSFactory().getFile(path);
+ if (f.exists() && !f.delete()) {
+ throw new RuntimeException("can not delete " + f.getAbsolutePath());
+ }
+
+ Schema schema = new Schema();
+
+ String sensorPrefix = "sensor_";
+ // the number of rows to include in the tablet
+ int rowNum = 1000000;
+ // the number of values to include in the tablet
+ int sensorNum = 10;
+
+ List<IMeasurementSchema> measurementSchemas = new ArrayList<>();
+ // add measurements into file schema (all with INT64 data type)
+ for (int i = 0; i < sensorNum; i++) {
+ IMeasurementSchema measurementSchema =
+ new MeasurementSchema(sensorPrefix + (i + 1), TSDataType.INT64, TSEncoding.TS_2DIFF);
+ measurementSchemas.add(measurementSchema);
+ schema.registerTimeseries(
+ new Path(device, sensorPrefix + (i + 1)),
+ new MeasurementSchema(sensorPrefix + (i + 1), TSDataType.INT64, TSEncoding.TS_2DIFF));
+ }
+
+ // add measurements into TSFileWriter
+ try (TsFileWriter tsFileWriter = new TsFileWriter(f, schema)) {
+
+ // construct the tablet
+ Tablet tablet = new Tablet(device, measurementSchemas);
+
+ long[] timestamps = tablet.timestamps;
+ Object[] values = tablet.values;
+
+ long timestamp = 1;
+ long value = 1000000L;
+
+ for (int r = 0; r < rowNum; r++, value++) {
+ int row = tablet.rowSize++;
+ timestamps[row] = timestamp++;
+ for (int i = 0; i < sensorNum; i++) {
+ long[] sensor = (long[]) values[i];
+ sensor[row] = value;
+ }
+ // write Tablet to TsFile
+ if (tablet.rowSize == tablet.getMaxRowNumber()) {
+ tsFileWriter.write(tablet);
+ tablet.reset();
+ }
+ }
+ // write Tablet to TsFile
+ if (tablet.rowSize != 0) {
+ tsFileWriter.write(tablet);
+ tablet.reset();
+ }
+ }
+ } catch (Exception e) {
+ throw new Exception("meet error in TsFileWrite with tablet", e);
+ }
+ }
+
+ @Test
+ public void tsFileSketchToolTest() {
+ TsFileSketchTool tool = new TsFileSketchTool();
+ String args[] = new String[2];
+ args[0] = path;
+ args[1] = sketchOut;
+ try {
+ tool.main(args);
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+
+ @After
+ public void tearDown() {
+ try {
+ FileUtils.forceDelete(new File(path));
+ FileUtils.forceDelete(new File(sketchOut));
+ } catch (IOException e) {
+ Assert.fail(e.getMessage());
+ }
+ }
+}
diff --git a/site/src/main/.vuepress/theme/global-components/Contributor.vue b/site/src/main/.vuepress/theme/global-components/Contributor.vue
index 2b631df..1f5e1cd 100644
--- a/site/src/main/.vuepress/theme/global-components/Contributor.vue
+++ b/site/src/main/.vuepress/theme/global-components/Contributor.vue
@@ -291,7 +291,7 @@
{
avatar:'/img/contributor-avatar/user.svg',
name: 'Stefanie Zhao',
- role: 'PPMC, Committer',
+ role: 'PMC, Committer',
id: 'zhaoxinyi',
date: '2018/11/18'
},