You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/04/13 09:05:57 UTC
[incubator-iotdb] 01/01: add meta
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch test_meta
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit f693533210ba279fcd1808efa2808f0ba6011483
Author: qiaojialin <64...@qq.com>
AuthorDate: Mon Apr 13 17:05:36 2020 +0800
add meta
---
.../main/java/org/apache/iotdb/SessionExample.java | 314 +++++----------------
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +-
.../tsfile/file/metadata/TimeseriesMetadata.java | 2 +
.../iotdb/tsfile/file/metadata/TsFileMetadata.java | 14 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 5 +
5 files changed, 98 insertions(+), 239 deletions(-)
diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
index 7f28c43..7c8f759 100644
--- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java
+++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java
@@ -18,271 +18,109 @@
*/
package org.apache.iotdb;
-import org.apache.iotdb.rpc.BatchExecutionException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.Session;
import org.apache.iotdb.session.SessionDataSet;
-import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.write.record.RowBatch;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.schema.Schema;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
public class SessionExample {
- private static Session session;
-
- public static void main(String[] args)
- throws IoTDBConnectionException, StatementExecutionException, BatchExecutionException {
- session = new Session("127.0.0.1", 6667, "root", "root");
- session.open(false);
-
- try {
- session.setStorageGroup("root.sg1");
- } catch (StatementExecutionException e) {
- if (!e.getMessage().contains("StorageGroupAlreadySetException")) {
- throw e;
- }
-// ignore duplicated set
- }
-
- if (!session.checkTimeseriesExists("root.sg1.d1.s1")) {
- session.createTimeseries("root.sg1.d1.s1", TSDataType.INT64, TSEncoding.RLE,
- CompressionType.SNAPPY);
- }
- if (!session.checkTimeseriesExists("root.sg1.d1.s2")) {
- session.createTimeseries("root.sg1.d1.s2", TSDataType.INT64, TSEncoding.RLE,
- CompressionType.SNAPPY);
- }
- if (!session.checkTimeseriesExists("root.sg1.d1.s3")) {
- session.createTimeseries("root.sg1.d1.s3", TSDataType.INT64, TSEncoding.RLE,
- CompressionType.SNAPPY);
+ public static void main(String[] args) {
+
+ for (int i = 0; i < 1; i++) {
+ new Thread(() -> {
+ try {
+ insert();
+ } catch (IoTDBConnectionException e) {
+ e.printStackTrace();
+ } catch (StatementExecutionException e) {
+ e.printStackTrace();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }).start();
+
+// try {
+// Thread.sleep(10000);
+// } catch (InterruptedException e) {
+// e.printStackTrace();
+// }
+//
+// new Thread(() -> {
+// try {
+// query();
+// } catch (IoTDBConnectionException e) {
+// e.printStackTrace();
+// } catch (StatementExecutionException e) {
+// e.printStackTrace();
+// }
+// }).start();
}
- insert();
- insertInBatch();
- insertRowBatch();
- nonQuery();
- query();
- deleteData();
- deleteTimeseries();
- session.close();
}
- private static void insert() throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = "root.sg1.d1";
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- for (long time = 0; time < 100; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
- session.insert(deviceId, time, measurements, values);
- }
- }
-
- private static void insertInObject()
- throws IoTDBConnectionException, StatementExecutionException {
- String deviceId = "root.sg1.d1";
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- for (long time = 0; time < 100; time++) {
- session.insert(deviceId, time, measurements, 1L, 1L, 1L);
- }
- }
+ private static void insert()
+ throws IoTDBConnectionException, StatementExecutionException, InterruptedException {
- private static void insertInBatch() throws IoTDBConnectionException, BatchExecutionException {
- String deviceId = "root.sg1.d1";
- List<String> measurements = new ArrayList<>();
- measurements.add("s1");
- measurements.add("s2");
- measurements.add("s3");
- List<String> deviceIds = new ArrayList<>();
- List<List<String>> measurementsList = new ArrayList<>();
- List<List<String>> valuesList = new ArrayList<>();
- List<Long> timestamps = new ArrayList<>();
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open();
- for (long time = 0; time < 500; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
+// while (true) {
+// Thread.sleep(5000);
+ long start = System.currentTimeMillis();
- deviceIds.add(deviceId);
- measurementsList.add(measurements);
- valuesList.add(values);
- timestamps.add(time);
- if (time != 0 && time % 100 == 0) {
- session.insertInBatch(deviceIds, timestamps, measurementsList, valuesList);
- deviceIds.clear();
- measurementsList.clear();
- valuesList.clear();
- timestamps.clear();
+ String deviceId = "root.sg1.d1";
+ List<String> measurements = new ArrayList<>();
+ for (int i = 0; i < 300000; i++) {
+ measurements.add("s" + i);
}
- }
-
- session.insertInBatch(deviceIds, timestamps, measurementsList, valuesList);
- }
-
- /**
- * insert a batch data of one device, each batch contains multiple timestamps with values of
- * sensors
- *
- * a RowBatch example:
- *
- * device1
- * time s1, s2, s3
- * 1, 1, 1, 1
- * 2, 2, 2, 2
- * 3, 3, 3, 3
- *
- * Users need to control the count of RowBatch and write a batch when it reaches the maxBatchSize
- */
- private static void insertRowBatch() throws IoTDBConnectionException, BatchExecutionException {
- // The schema of sensors of one device
- Schema schema = new Schema();
- schema.registerTimeseries(new Path("root.sg1.d1", "s1"),
- new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- schema.registerTimeseries(new Path("root.sg1.d1", "s2"),
- new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
- schema.registerTimeseries(new Path("root.sg1.d1", "s3"),
- new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
-
- RowBatch rowBatch = schema.createRowBatch("root.sg1.d1", 100);
- long[] timestamps = rowBatch.timestamps;
- Object[] values = rowBatch.values;
-
- for (long time = 0; time < 100; time++) {
- int row = rowBatch.batchSize++;
- timestamps[row] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor = (long[]) values[i];
- sensor[row] = i;
- }
- if (rowBatch.batchSize == rowBatch.getMaxBatchSize()) {
- session.insertBatch(rowBatch);
- rowBatch.reset();
+ List<String> values = new ArrayList<>();
+ for (int i = 0; i < 300000; i++) {
+ values.add("1");
}
- }
- if (rowBatch.batchSize != 0) {
- session.insertBatch(rowBatch);
- rowBatch.reset();
- }
+ session.insert(deviceId, start, measurements, values);
+ System.out.println(
+ Thread.currentThread().getName() + " write: " + (System.currentTimeMillis() - start));
+// }
}
- private static void insertMultipleDeviceRowBatch()
- throws IoTDBConnectionException, BatchExecutionException {
- // The schema of sensors of one device
- Schema schema1 = new Schema();
- schema1.registerTimeseries(new Path("root.sg1.d1.s1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- schema1.registerTimeseries(new Path("root.sg1.d1.s2"), new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
- schema1.registerTimeseries(new Path("root.sg1.d1.s3"), new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
-
- RowBatch rowBatch1 = schema1.createRowBatch("root.sg1.d1", 100);
-
- Schema schema2 = new Schema();
- schema2.registerTimeseries(new Path("root.sg1.d2.s1"), new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- schema2.registerTimeseries(new Path("root.sg1.d2.s2"), new MeasurementSchema("s2", TSDataType.INT64, TSEncoding.RLE));
- schema2.registerTimeseries(new Path("root.sg1.d2.s3"), new MeasurementSchema("s3", TSDataType.INT64, TSEncoding.RLE));
-
- RowBatch rowBatch2 = schema2.createRowBatch("root.sg1.d2", 100);
-
- Schema schema3 = new Schema();
- Map<String, MeasurementSchema> template = new HashMap<>();
- template.put("s1", new MeasurementSchema("s1", TSDataType.INT64, TSEncoding.RLE));
- template.put("s2", new MeasurementSchema("s2", TSDataType.INT32, TSEncoding.RLE));
- schema3.registerDeviceTemplate("template3", template);
- schema3.extendTemplate("template3", new MeasurementSchema("s3", TSDataType.FLOAT, TSEncoding.RLE));
- schema3.registerDevice("root.sg1.d3", "template3");
-
- RowBatch rowBatch3 = schema3.createRowBatch("root.sg1.d3", 100);
-
- Map<String, RowBatch> rowBatchMap = new HashMap<>();
- rowBatchMap.put("root.sg1.d1", rowBatch1);
- rowBatchMap.put("root.sg1.d2", rowBatch2);
- rowBatchMap.put("root.sg1.d3", rowBatch3);
-
- long[] timestamps1 = rowBatch1.timestamps;
- Object[] values1 = rowBatch1.values;
- long[] timestamps2 = rowBatch2.timestamps;
- Object[] values2 = rowBatch2.values;
- long[] timestamps3 = rowBatch3.timestamps;
- Object[] values3 = rowBatch3.values;
-
- for (long time = 0; time < 100; time++) {
- int row1 = rowBatch1.batchSize++;
- int row2 = rowBatch2.batchSize++;
- int row3 = rowBatch3.batchSize++;
- timestamps1[row1] = time;
- timestamps2[row2] = time;
- timestamps3[row3] = time;
- for (int i = 0; i < 3; i++) {
- long[] sensor1 = (long[]) values1[i];
- sensor1[row1] = i;
- long[] sensor2 = (long[]) values2[i];
- sensor2[row2] = i;
- long[] sensor3 = (long[]) values3[i];
- sensor3[row3] = i;
+ private static void query() throws IoTDBConnectionException, StatementExecutionException {
+ SessionDataSet dataSet;
+ Session session = new Session("127.0.0.1", 6667, "root", "root");
+ session.open();
+
+ while (true) {
+ try {
+ Thread.sleep(5000);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
}
- if (rowBatch1.batchSize == rowBatch1.getMaxBatchSize()) {
- session.insertMultipleDeviceBatch(rowBatchMap);
+ long start = System.currentTimeMillis();
- rowBatch1.reset();
- rowBatch2.reset();
- rowBatch3.reset();
+ StringBuilder builder = new StringBuilder("select last ");
+ for (int c = 0; c < 49999; c++) {
+ builder.append("s").append(c).append(",");
}
- }
-
- if (rowBatch1.batchSize != 0) {
- session.insertMultipleDeviceBatch(rowBatchMap);
- rowBatch1.reset();
- rowBatch2.reset();
- rowBatch3.reset();
- }
- }
- private static void deleteData() throws IoTDBConnectionException, StatementExecutionException {
- String path = "root.sg1.d1.s1";
- long deleteTime = 99;
- session.deleteData(path, deleteTime);
- }
+ builder.append("s49999 ");
+ builder.append("from root.sg1.d1");
- private static void deleteTimeseries()
- throws IoTDBConnectionException, StatementExecutionException {
- List<String> paths = new ArrayList<>();
- paths.add("root.sg1.d1.s1");
- paths.add("root.sg1.d1.s2");
- paths.add("root.sg1.d1.s3");
- session.deleteTimeseries(paths);
- }
-
- private static void query() throws IoTDBConnectionException, StatementExecutionException {
- SessionDataSet dataSet;
- dataSet = session.executeQueryStatement("select * from root.sg1.d1");
- System.out.println(dataSet.getColumnNames());
- dataSet.setBatchSize(1024); // default is 512
- while (dataSet.hasNext()) {
- System.out.println(dataSet.next());
+ dataSet = session.executeQueryStatement(builder.toString());
+// System.out.println(dataSet.getColumnNames());
+ int a = 0;
+ while (dataSet.hasNext()) {
+ a++;
+ dataSet.next();
+ }
+ System.out.print(Thread.currentThread().getName() + " read " + a + " ");
+ System.out.println(System.currentTimeMillis() - start);
+ dataSet.closeOperationHandle();
}
- dataSet.closeOperationHandle();
}
- private static void nonQuery() throws IoTDBConnectionException, StatementExecutionException {
- session.executeNonQueryStatement("insert into root.sg1.d1(timestamp,s1) values(200, 1);");
- }
}
\ No newline at end of file
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index a1dee4d..9971bb6 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -118,7 +118,7 @@ public class IoTDBConfig {
/**
* Is dynamic parameter adapter enable.
*/
- private boolean enableParameterAdapter = true;
+ private boolean enableParameterAdapter = false;
/**
* Is the write ahead log enable.
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
index f6051bf..c25fc8b 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TimeseriesMetadata.java
@@ -19,6 +19,8 @@
package org.apache.iotdb.tsfile.file.metadata;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.statistics.Statistics;
import org.apache.iotdb.tsfile.read.controller.IChunkMetadataLoader;
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
index feb147e..9c35d00 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/TsFileMetadata.java
@@ -90,6 +90,15 @@ public class TsFileMetadata {
}
fileMetaData.setVersionInfo(versionInfo);
+ long start = System.currentTimeMillis();
+ Map<String, Long> map = new HashMap<>();
+ for (int i = 0; i < 300000; i++) {
+ String key = ReadWriteIOUtils.readString(buffer);
+ Long value = ReadWriteIOUtils.readLong(buffer);
+ map.put(key + i, value);
+ }
+ System.out.println("read String " + map.size());
+ System.out.println(System.currentTimeMillis() - start);
// read bloom filter
if (buffer.hasRemaining()) {
@@ -138,6 +147,11 @@ public class TsFileMetadata {
byteLen +=ReadWriteIOUtils.write(versionPair.right, outputStream);
}
+ for (int i = 0; i < 300000; i++) {
+ byteLen += ReadWriteIOUtils.write("root.XDB.SDD.tag50000000", outputStream);
+ byteLen += ReadWriteIOUtils.write(100000L, outputStream);
+ }
+
return byteLen;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index eb0c6ae..5240489 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -296,6 +296,8 @@ public class TsFileSequenceReader implements AutoCloseable {
if (!tsFileMetaData.getDeviceMetadataIndex().containsKey(device)) {
return Collections.emptyMap();
}
+
+ long start = System.currentTimeMillis();
Pair<Long, Integer> deviceMetadataIndex = tsFileMetaData.getDeviceMetadataIndex().get(device);
Map<String, TimeseriesMetadata> deviceMetadata = new HashMap<>();
ByteBuffer buffer = readData(deviceMetadataIndex.left, deviceMetadataIndex.right);
@@ -303,6 +305,9 @@ public class TsFileSequenceReader implements AutoCloseable {
TimeseriesMetadata tsMetaData = TimeseriesMetadata.deserializeFrom(buffer);
deviceMetadata.put(tsMetaData.getMeasurementId(), tsMetaData);
}
+
+ System.out.println("read TimeseriesMetadata " + deviceMetadata.size());
+ System.out.println(System.currentTimeMillis() - start);
return deviceMetadata;
}