You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/13 09:05:56 UTC
[incubator-iotdb] branch test_meta created (now f693533)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a change to branch test_meta
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
at f693533 add meta
This branch includes the following new commits:
new f693533 add meta
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[incubator-iotdb] 01/01: add meta
Posted by qi...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch test_meta
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f693533210ba279fcd1808efa2808f0ba6011483
Author: qiaojialin <64...@qq.com>
AuthorDate: Mon Apr 13 17:05:36 2020 +0800
add meta
---
.../main/java/org/apache/iotdb/SessionExample.java | 314 +++++----------------
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 2 +
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 14 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 5 +
5 files changed, 98 insertions(+), 239 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index 7f28c43..7c8f759 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -18,271 +18,109 @@
*/
package org.apache.iotdb;
-import org.apache.iotdb.rpc.BatchExecutionException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.Session;
import org.apache.iotdb.session.SessionDataSet;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.write.record.RowBatch;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.schema.Schema;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
public class SessionExample {
- private static Session session;
-
- public static void main(String[] args)
- throws IoTDBConnectionException, StatementExecutionException, BatchExecutionException {
- session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
-
- try {
- session.setStorageGroup("root.sg1");
- } catch (StatementExecutionException e) {
- if (!e.getMessage().contains("StorageGroupAlreadySetException")) {
- throw e;
- }
-// ignore duplicated set
- }
-
- if (!session.checkTimeseriesExists("root.sg1.d1.s1")) {
- session.createTimeseries("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE,
- CompressionType.SNAPPY);
- }
- if (!session.checkTimeseriesExists("root.sg1.d1.s2")) {
- session.createTimeseries("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE,
- CompressionType.SNAPPY);
- }
- if (!session.checkTimeseriesExists("root.sg1.d1.s3")) {
- session.createTimeseries("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE,
- CompressionType.SNAPPY);
+ public static void main(String[] args) {
+
+ for (int i = 0; i < 1; i++) {
+ new Thread(() -> {
+ try {
+ insert();
+ } catch (IoTDBConnectionException e) {
+ e.printStackTrace();
+ } catch (StatementExecutionException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }).start();
+
+// try {
+// Thread.sleep(10000);
+// } catch (InterruptedException e) {
+// e.printStackTrace();
+// }
+//
+// new Thread(() -> {
+// try {
+// query();
+// } catch (IoTDBConnectionException e) {
+// e.printStackTrace();
+// } catch (StatementExecutionException e) {
+// e.printStackTrace();
+// }
+// }).start();
}
- insert();
- insertInBatch();
- insertRowBatch();
- nonQuery();
- query();
- deleteData();
- deleteTimeseries();
- session.close();
}
- private static void insert() throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = "root.sg1.d1";
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- for (long time = 0; time < 100; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
- session.insert(deviceId, time, measurements, values);
- }
- }
-
- private static void insertInObject()
- throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = "root.sg1.d1";
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- for (long time = 0; time < 100; time++) {
- session.insert(deviceId, time, measurements, 1L, 1L, 1L);
- }
- }
+ private static void insert()
+ throws IoTDBConnectionException, StatementExecutionException, InterruptedException {
- private static void insertInBatch() throws IoTDBConnectionException, BatchExecutionException {
- String deviceId = "root.sg1.d1";
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- List<String> deviceIds = new ArrayList<>();
- List<List<String>> measurementsList = new ArrayList<>();
- List<List<String>> valuesList = new ArrayList<>();
- List<Long> timestamps = new ArrayList<>();
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open();
- for (long time = 0; time < 500; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
+// while (true) {
+// Thread.sleep(5000);
+ long start = System.currentTimeMillis();
- deviceIds.add(deviceId);
- measurementsList.add(measurements);
- valuesList.add(values);
- timestamps.add(time);
- if (time != 0 && time % 100 == 0) {
- session.insertInBatch(deviceIds, timestamps, measurementsList, valuesList);
- deviceIds.clear();
- measurementsList.clear();
- valuesList.clear();
- timestamps.clear();
+ String deviceId = "root.sg1.d1";
+ List<String> measurements = new ArrayList<>();
+ for (int i = 0; i < 300000; i++) {
+ measurements.add("s" + i);
}
- }
-
- session.insertInBatch(deviceIds, timestamps, measurementsList, valuesList);
- }
-
- /**
- * insert a batch data of one device, each batch contains multiple timestamps with values of
- * sensors
- *
- * a RowBatch example:
- *
- * device1
- * time s1, s2, s3
- * 1, 1, 1, 1
- * 2, 2, 2, 2
- * 3, 3, 3, 3
- *
- * Users need to control the count of RowBatch and write a batch when it reaches the maxBatchSize
- */
- private static void insertRowBatch() throws IoTDBConnectionException, BatchExecutionException {
- // The schema of sensors of one device
- Schema schema = new Schema();
- schema.registerTimeseries(new Path("root.sg1.d1", "s1"),
- new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- schema.registerTimeseries(new Path("root.sg1.d1", "s2"),
- new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
- schema.registerTimeseries(new Path("root.sg1.d1", "s3"),
- new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
-
- RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100);
- long[] timestamps = rowBatch.timestamps;
- Object[] values = rowBatch.values;
-
- for (long time = 0; time < 100; time++) {
- int row = rowBatch.batchSize++;
- timestamps[row] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor = (long[]) values[i];
- sensor[row] = i;
- }
- if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) {
- session.insertBatch(rowBatch);
- rowBatch.reset();
+ List<String> values = new ArrayList<>();
+ for (int i = 0; i < 300000; i++) {
+ values.add("1");
}
- }
- if (rowBatch.batchSize != 0) {
- session.insertBatch(rowBatch);
- rowBatch.reset();
- }
+ session.insert(deviceId, start, measurements, values);
+ System.out.println(
+ Thread.currentThread().getName() + " write: " + (System.currentTimeMillis() - start));
+// }
}
- private static void insertMultipleDeviceRowBatch()
- throws IoTDBConnectionException, BatchExecutionException {
- // The schema of sensors of one device
- Schema schema1 = new Schema();
- schema1.registerTimeseries(new Path("root.sg1.d1.s1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- schema1.registerTimeseries(new Path("root.sg1.d1.s2"), new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
- schema1.registerTimeseries(new Path("root.sg1.d1.s3"), new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
-
- RowBatch rowBatch1 = schema1.createRowBatch("root.sg1.d1", 100);
-
- Schema schema2 = new Schema();
- schema2.registerTimeseries(new Path("root.sg1.d2.s1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- schema2.registerTimeseries(new Path("root.sg1.d2.s2"), new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
- schema2.registerTimeseries(new Path("root.sg1.d2.s3"), new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
-
- RowBatch rowBatch2 = schema2.createRowBatch("root.sg1.d2", 100);
-
- Schema schema3 = new Schema();
- Map<String, MeasurementSchema> template = new HashMap<>();
- template.put("s1", new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- template.put("s2", new MeasurementSchema("s2", TSDataType.INT32, TSEncoding.RLE));
- schema3.registerDeviceTemplate("template3", template);
- schema3.extendTemplate("template3", new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.RLE));
- schema3.registerDevice("root.sg1.d3", "template3");
-
- RowBatch rowBatch3 = schema3.createRowBatch("root.sg1.d3", 100);
-
- Map<String, RowBatch> rowBatchMap = new HashMap<>();
- rowBatchMap.put("root.sg1.d1", rowBatch1);
- rowBatchMap.put("root.sg1.d2", rowBatch2);
- rowBatchMap.put("root.sg1.d3", rowBatch3);
-
- long[] timestamps1 = rowBatch1.timestamps;
- Object[] values1 = rowBatch1.values;
- long[] timestamps2 = rowBatch2.timestamps;
- Object[] values2 = rowBatch2.values;
- long[] timestamps3 = rowBatch3.timestamps;
- Object[] values3 = rowBatch3.values;
-
- for (long time = 0; time < 100; time++) {
- int row1 = rowBatch1.batchSize++;
- int row2 = rowBatch2.batchSize++;
- int row3 = rowBatch3.batchSize++;
- timestamps1[row1] = time;
- timestamps2[row2] = time;
- timestamps3[row3] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor1 = (long[]) values1[i];
- sensor1[row1] = i;
- long[] sensor2 = (long[]) values2[i];
- sensor2[row2] = i;
- long[] sensor3 = (long[]) values3[i];
- sensor3[row3] = i;
+ private static void query() throws IoTDBConnectionException, StatementExecutionException {
+ SessionDataSet dataSet;
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open();
+
+ while (true) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
}
- if (rowBatch1.batchSize == rowBatch1.getMaxBatchSize()) {
- session.insertMultipleDeviceBatch(rowBatchMap);
+ long start = System.currentTimeMillis();
- rowBatch1.reset();
- rowBatch2.reset();
- rowBatch3.reset();
+ StringBuilder builder = new StringBuilder("select last ");
+ for (int c = 0; c < 49999; c++) {
+ builder.append("s").append(c).append(",");
}
- }
-
- if (rowBatch1.batchSize != 0) {
- session.insertMultipleDeviceBatch(rowBatchMap);
- rowBatch1.reset();
- rowBatch2.reset();
- rowBatch3.reset();
- }
- }
- private static void deleteData() throws IoTDBConnectionException, StatementExecutionException {
- String path = "root.sg1.d1.s1";
- long deleteTime = 99;
- session.deleteData(path, deleteTime);
- }
+ builder.append("s49999 ");
+ builder.append("from root.sg1.d1");
- private static void deleteTimeseries()
- throws IoTDBConnectionException, StatementExecutionException {
- List<String> paths = new ArrayList<>();
- paths.add("root.sg1.d1.s1");
- paths.add("root.sg1.d1.s2");
- paths.add("root.sg1.d1.s3");
- session.deleteTimeseries(paths);
- }
-
- private static void query() throws IoTDBConnectionException, StatementExecutionException {
- SessionDataSet dataSet;
- dataSet = session.executeQueryStatement("select * from root.sg1.d1");
- System.out.println(dataSet.getColumnNames());
- dataSet.setBatchSize(1024); // default is 512
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
+ dataSet = session.executeQueryStatement(builder.toString());
+// System.out.println(dataSet.getColumnNames());
+ int a = 0;
+ while (dataSet.hasNext()) {
+ a++;
+ dataSet.next();
+ }
+ System.out.print(Thread.currentThread().getName() + " read " + a + " ");
+ System.out.println(System.currentTimeMillis() - start);
+ dataSet.closeOperationHandle();
}
- dataSet.closeOperationHandle();
}
- private static void nonQuery() throws IoTDBConnectionException, StatementExecutionException {
- session.executeNonQueryStatement("insert into root.sg1.d1(timestamp,s1) values(200, 1);");
- }
}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index a1dee4d..9971bb6 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -118,7 +118,7 @@ public class IoTDBConfig {
/**
* Is dynamic parameter adapter enable.
*/
- private boolean enableParameterAdapter = true;
+ private boolean enableParameterAdapter = false;
/**
* Is the write ahead log enable.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index f6051bf..c25fc8b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.tsfile.file.metadata;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
index feb147e..9c35d00 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
@@ -90,6 +90,15 @@ public class TsFileMetadata {
}
fileMetaData.setVersionInfo(versionInfo);
+ long start = System.currentTimeMillis();
+ Map<String, Long> map = new HashMap<>();
+ for (int i = 0; i < 300000; i++) {
+ String key = ReadWriteIOUtils.readString(buffer);
+ Long value = ReadWriteIOUtils.readLong(buffer);
+ map.put(key + i, value);
+ }
+ System.out.println("read String " + map.size());
+ System.out.println(System.currentTimeMillis() - start);
// read bloom filter
if (buffer.hasRemaining()) {
@@ -138,6 +147,11 @@ public class TsFileMetadata {
byteLen +=ReadWriteIOUtils.write(versionPair.right, outputStream);
}
+ for (int i = 0; i < 300000; i++) {
+ byteLen += ReadWriteIOUtils.write("root.XDB.SDD.tag50000000", outputStream);
+ byteLen += ReadWriteIOUtils.write(100000L, outputStream);
+ }
+
return byteLen;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index eb0c6ae..5240489 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -296,6 +296,8 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!tsFileMetaData.getDeviceMetadataIndex().containsKey(device)) {
return Collections.emptyMap();
}
+
+ long start = System.currentTimeMillis();
Pair<Long, Integer> deviceMetadataIndex = tsFileMetaData.getDeviceMetadataIndex().get(device);
Map<String, TimeseriesMetadata> deviceMetadata = new HashMap<>();
ByteBuffer buffer = readData(deviceMetadataIndex.left, deviceMetadataIndex.right);
@@ -303,6 +305,9 @@ public class TsFileSequenceReader implements AutoCloseable {
TimeseriesMetadata tsMetaData = TimeseriesMetadata.deserializeFrom(buffer);
deviceMetadata.put(tsMetaData.getMeasurementId(), tsMetaData);
}
+
+ System.out.println("read TimeseriesMetadata " + deviceMetadata.size());
+ System.out.println(System.currentTimeMillis() - start);
return deviceMetadata;
}