You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@iotdb.apache.org by GitBox <gi...@apache.org> on 2020/05/20 07:27:24 UTC
[GitHub] [incubator-iotdb] SilverNarcissus commented on a change in pull request #1229: [IOTDB-615] Use binary rather than string in insert plan
SilverNarcissus commented on a change in pull request #1229:
URL: https://github.com/apache/incubator-iotdb/pull/1229#discussion_r427795850
##########
File path: example/session/src/main/java/org/apache/iotdb/SessionExample.java
##########
@@ -168,32 +178,47 @@ private static void insertRecords() throws IoTDBConnectionException, BatchExecut
measurements.add("s3");
List<String> deviceIds = new ArrayList<>();
List<List<String>> measurementsList = new ArrayList<>();
- List<List<String>> valuesList = new ArrayList<>();
+ List<List<Object>> valuesList = new ArrayList<>();
List<Long> timestamps = new ArrayList<>();
+ List<List<TSDataType>> typesList = new ArrayList<>();
for (long time = 0; time < 500; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
+ List<Object> values = new ArrayList<>();
+ List<TSDataType> types = new ArrayList<>();
+ values.add(1L);
+ values.add(2L);
+ values.add(3L);
+ types.add(TSDataType.INT64);
+ types.add(TSDataType.INT64);
+ types.add(TSDataType.INT64);
deviceIds.add(deviceId);
measurementsList.add(measurements);
valuesList.add(values);
+ typesList.add(types);
timestamps.add(time);
if (time != 0 && time % 100 == 0) {
- session.insertRecords(deviceIds, timestamps, measurementsList, valuesList);
+ session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
deviceIds.clear();
measurementsList.clear();
valuesList.clear();
timestamps.clear();
}
}
- session.insertRecords(deviceIds, timestamps, measurementsList, valuesList);
+ session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
}
-
/**
+<<<<<<< HEAD
+ * insert a batch data of one device, each batch contains multiple timestamps with values of
+ * sensors
+ * <p>
+ * a RowBatch example:
+ * <p>
+ * device1 time s1, s2, s3 1, 1, 1, 1 2, 2, 2, 2 3, 3, 3, 3
Review comment:
Fixed~
##########
File path: example/session/src/main/java/org/apache/iotdb/SessionExample.java
##########
@@ -168,32 +178,47 @@ private static void insertRecords() throws IoTDBConnectionException, BatchExecut
measurements.add("s3");
List<String> deviceIds = new ArrayList<>();
List<List<String>> measurementsList = new ArrayList<>();
- List<List<String>> valuesList = new ArrayList<>();
+ List<List<Object>> valuesList = new ArrayList<>();
List<Long> timestamps = new ArrayList<>();
+ List<List<TSDataType>> typesList = new ArrayList<>();
for (long time = 0; time < 500; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
+ List<Object> values = new ArrayList<>();
+ List<TSDataType> types = new ArrayList<>();
+ values.add(1L);
+ values.add(2L);
+ values.add(3L);
+ types.add(TSDataType.INT64);
+ types.add(TSDataType.INT64);
+ types.add(TSDataType.INT64);
deviceIds.add(deviceId);
measurementsList.add(measurements);
valuesList.add(values);
+ typesList.add(types);
timestamps.add(time);
if (time != 0 && time % 100 == 0) {
- session.insertRecords(deviceIds, timestamps, measurementsList, valuesList);
+ session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
deviceIds.clear();
measurementsList.clear();
valuesList.clear();
timestamps.clear();
}
}
- session.insertRecords(deviceIds, timestamps, measurementsList, valuesList);
+ session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
}
-
/**
+<<<<<<< HEAD
+ * insert a batch data of one device, each batch contains multiple timestamps with values of
+ * sensors
+ * <p>
+ * a RowBatch example:
+ * <p>
+ * device1 time s1, s2, s3 1, 1, 1, 1 2, 2, 2, 2 3, 3, 3, 3
+ * <p>
+ * Users need to control the count of RowBatch and write a batch when it reaches the maxBatchSize
+=======
Review comment:
Fixed~
##########
File path: example/session/src/main/java/org/apache/iotdb/SessionExample.java
##########
@@ -205,6 +230,7 @@ private static void insertRecords() throws IoTDBConnectionException, BatchExecut
* 3, 3, 3, 3
*
* Users need to control the count of Tablet and write a batch when it reaches the maxBatchSize
+>>>>>>> 73dd5eb6e44e2194b650eea72aabcfaf929e5bdc
Review comment:
Fixed~
##########
File path: example/session/src/main/java/org/apache/iotdb/SessionExample.java
##########
@@ -168,32 +178,47 @@ private static void insertRecords() throws IoTDBConnectionException, BatchExecut
measurements.add("s3");
List<String> deviceIds = new ArrayList<>();
List<List<String>> measurementsList = new ArrayList<>();
- List<List<String>> valuesList = new ArrayList<>();
+ List<List<Object>> valuesList = new ArrayList<>();
List<Long> timestamps = new ArrayList<>();
+ List<List<TSDataType>> typesList = new ArrayList<>();
for (long time = 0; time < 500; time++) {
- List<String> values = new ArrayList<>();
- values.add("1");
- values.add("2");
- values.add("3");
+ List<Object> values = new ArrayList<>();
+ List<TSDataType> types = new ArrayList<>();
+ values.add(1L);
+ values.add(2L);
+ values.add(3L);
+ types.add(TSDataType.INT64);
+ types.add(TSDataType.INT64);
+ types.add(TSDataType.INT64);
deviceIds.add(deviceId);
measurementsList.add(measurements);
valuesList.add(values);
+ typesList.add(types);
timestamps.add(time);
if (time != 0 && time % 100 == 0) {
- session.insertRecords(deviceIds, timestamps, measurementsList, valuesList);
+ session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
deviceIds.clear();
measurementsList.clear();
valuesList.clear();
timestamps.clear();
}
}
- session.insertRecords(deviceIds, timestamps, measurementsList, valuesList);
+ session.insertRecords(deviceIds, timestamps, measurementsList, typesList, valuesList);
}
-
/**
+<<<<<<< HEAD
Review comment:
Fixed~
##########
File path: server/src/main/java/org/apache/iotdb/db/qp/physical/crud/InsertPlan.java
##########
@@ -174,8 +234,102 @@ public void serializeTo(DataOutputStream stream) throws IOException {
schema.serializeTo(stream);
}
- for (String m : values) {
- putString(stream, m);
+ try {
+ putValues(stream);
+ } catch (QueryProcessException e) {
+ throw new IOException(e);
+ }
+ }
+
+ private void putValues(DataOutputStream outputStream) throws QueryProcessException, IOException {
+ for (int i = 0; i < values.length; i++) {
+ ReadWriteIOUtils.write(types[i], outputStream);
+ switch (types[i]) {
+ case BOOLEAN:
+ ReadWriteIOUtils.write((Boolean) values[i], outputStream);
+ break;
+ case INT32:
+ ReadWriteIOUtils.write((Integer) values[i], outputStream);
+ break;
+ case INT64:
+ ReadWriteIOUtils.write((Long) values[i], outputStream);
+ break;
+ case FLOAT:
+ ReadWriteIOUtils.write((Float) values[i], outputStream);
+ break;
+ case DOUBLE:
+ ReadWriteIOUtils.write((Double) values[i], outputStream);
+ break;
+ case TEXT:
+ ReadWriteIOUtils.write((Binary) values[i], outputStream);
+ break;
+ default:
+ throw new QueryProcessException("Unsupported data type:" + types[i]);
+ }
+ }
+ }
+
+ private void putValues(ByteBuffer buffer) throws QueryProcessException {
+ for (int i = 0; i < values.length; i++) {
+ ReadWriteIOUtils.write(types[i], buffer);
+ switch (types[i]) {
+ case BOOLEAN:
+ ReadWriteIOUtils.write((Boolean) values[i], buffer);
+ break;
+ case INT32:
+ ReadWriteIOUtils.write((Integer) values[i], buffer);
+ break;
+ case INT64:
+ ReadWriteIOUtils.write((Long) values[i], buffer);
+ break;
+ case FLOAT:
+ ReadWriteIOUtils.write((Float) values[i], buffer);
+ break;
+ case DOUBLE:
+ ReadWriteIOUtils.write((Double) values[i], buffer);
+ break;
+ case TEXT:
+ ReadWriteIOUtils.write((Binary) values[i], buffer);
+ break;
+ default:
+ throw new QueryProcessException("Unsupported data type:" + types[i]);
+ }
+ }
+ }
+
+ public TSDataType[] getTypes() {
+ return types;
+ }
+
+ public void setTypes(TSDataType[] types) {
+ this.types = types;
+ }
+
+ public void getValues(ByteBuffer buffer) throws QueryProcessException {
Review comment:
Sure~
##########
File path: server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
##########
@@ -1111,7 +1118,9 @@ public TSStatus insertRecord(TSInsertRecordReq req) {
plan.setDeviceId(req.getDeviceId());
plan.setTime(req.getTimestamp());
plan.setMeasurements(req.getMeasurements().toArray(new String[0]));
- plan.setValues(req.getValues().toArray(new String[0]));
+ plan.setTypes(new TSDataType[plan.getMeasurements().length]);
+ plan.setValues(new Object[plan.getMeasurements().length]);
+ plan.getValues(req.values);
Review comment:
Fixed~
##########
File path: session/src/main/java/org/apache/iotdb/session/Session.java
##########
@@ -393,16 +359,17 @@ public void insertTablets(Map<String, Tablet> tablets, boolean sorted)
}
/**
- * Insert multiple rows, which can reduce the overhead of network. This method is just like
- * jdbc executeBatch, we pack some insert request in batch and send them to server.
- * If you want improve your performance, please see insertTablet method
- *
+ * Insert multiple rows, which can reduce the overhead of network. This method is just like jdbc
+ * executeBatch, we pack some insert request in batch and send them to server. If you want improve
+ * your performance, please see insertTablet method
+ * <p>
* Each row is independent, which could have different deviceId, time, number of measurements
*
* @see Session#insertTablet(Tablet)
*/
public void insertRecords(List<String> deviceIds, List<Long> times,
- List<List<String>> measurementsList, List<List<String>> valuesList)
+ List<List<String>> measurementsList, List<List<TSDataType>> typesList,
+ List<List<Object>> valuesList)
Review comment:
If user uses wrong type, we will fall at server end. I think we need fall at client to follow the "fall quickly" principle.
##########
File path: session/src/main/java/org/apache/iotdb/session/Session.java
##########
@@ -215,66 +218,29 @@ private boolean reconnect() {
}
/**
- * insert data in one row, if you want to improve your performance, please use insertRecords method
- * or insertTablet method
+ * insert data in one row, if you want to improve your performance, please use insertRecords
+ * method or insertTablet method
*
- * @see Session#insertRecords(List, List, List, List)
+ * @see Session#insertRecords(List, List, List, List, List)
* @see Session#insertTablet(Tablet)
*/
public void insertRecord(String deviceId, long time, List<String> measurements,
+ List<TSDataType> types,
Object... values) throws IoTDBConnectionException, StatementExecutionException {
- List<String> stringValues = new ArrayList<>();
- for (Object o : values) {
- stringValues.add(o.toString());
- }
+ List<Object> valuesList = new ArrayList<>(Arrays.asList(values));
- insertRecord(deviceId, time, measurements, stringValues);
+ insertRecord(deviceId, time, measurements, types, valuesList);
}
- /**
- * insert data in one row, if you want to improve your performance, please use insertRecords method
- * or insertTablet method
- *
- * @see Session#insertRecords(List, List, List, List)
- * @see Session#insertTablet(Tablet)
- */
- public void insertRecord(String deviceId, long time, List<String> measurements,
- List<String> values) throws IoTDBConnectionException, StatementExecutionException {
- TSInsertRecordReq request = new TSInsertRecordReq();
- request.setSessionId(sessionId);
- request.setDeviceId(deviceId);
- request.setTimestamp(time);
- request.setMeasurements(measurements);
- request.setValues(values);
-
- try {
- RpcUtils.verifySuccess(client.insertRecord(request));
- } catch (TException e) {
- if (reconnect()) {
- try {
- RpcUtils.verifySuccess(client.insertRecord(request));
- } catch (TException e1) {
- throw new IoTDBConnectionException(e1);
- }
- } else {
- throw new IoTDBConnectionException("Fail to reconnect to server,"
- + " please check server status", e);
- }
- }
- }
/**
* insert the data of a device. For each timestamp, the number of measurements is the same.
- *
- * a Tablet example:
- *
- * device1
- * time s1, s2, s3
- * 1, 1, 1, 1
- * 2, 2, 2, 2
- * 3, 3, 3, 3
- *
+ * <p>
+ * a Tablet example:
+ * <p>
+ * device1 time s1, s2, s3 1, 1, 1, 1 2, 2, 2, 2 3, 3, 3, 3
+ * <p>
Review comment:
Sure~
##########
File path: session/src/main/java/org/apache/iotdb/session/Session.java
##########
@@ -215,66 +218,29 @@ private boolean reconnect() {
}
/**
- * insert data in one row, if you want to improve your performance, please use insertRecords method
- * or insertTablet method
+ * insert data in one row, if you want to improve your performance, please use insertRecords
+ * method or insertTablet method
*
- * @see Session#insertRecords(List, List, List, List)
+ * @see Session#insertRecords(List, List, List, List, List)
* @see Session#insertTablet(Tablet)
*/
public void insertRecord(String deviceId, long time, List<String> measurements,
+ List<TSDataType> types,
Object... values) throws IoTDBConnectionException, StatementExecutionException {
Review comment:
This method is add for some client using dynamic parameter.
##########
File path: session/src/main/java/org/apache/iotdb/session/Session.java
##########
@@ -215,66 +218,29 @@ private boolean reconnect() {
}
/**
- * insert data in one row, if you want to improve your performance, please use insertRecords method
- * or insertTablet method
+ * insert data in one row, if you want to improve your performance, please use insertRecords
+ * method or insertTablet method
*
- * @see Session#insertRecords(List, List, List, List)
+ * @see Session#insertRecords(List, List, List, List, List)
* @see Session#insertTablet(Tablet)
*/
public void insertRecord(String deviceId, long time, List<String> measurements,
+ List<TSDataType> types,
Object... values) throws IoTDBConnectionException, StatementExecutionException {
- List<String> stringValues = new ArrayList<>();
- for (Object o : values) {
- stringValues.add(o.toString());
- }
+ List<Object> valuesList = new ArrayList<>(Arrays.asList(values));
- insertRecord(deviceId, time, measurements, stringValues);
+ insertRecord(deviceId, time, measurements, types, valuesList);
}
- /**
- * insert data in one row, if you want to improve your performance, please use insertRecords method
- * or insertTablet method
- *
- * @see Session#insertRecords(List, List, List, List)
- * @see Session#insertTablet(Tablet)
- */
- public void insertRecord(String deviceId, long time, List<String> measurements,
Review comment:
Thanks~ Could you give a function signature here?
##########
File path: session/src/main/java/org/apache/iotdb/session/Session.java
##########
@@ -215,66 +218,29 @@ private boolean reconnect() {
}
/**
- * insert data in one row, if you want to improve your performance, please use insertRecords method
- * or insertTablet method
+ * insert data in one row, if you want to improve your performance, please use insertRecords
+ * method or insertTablet method
*
- * @see Session#insertRecords(List, List, List, List)
+ * @see Session#insertRecords(List, List, List, List, List)
* @see Session#insertTablet(Tablet)
*/
public void insertRecord(String deviceId, long time, List<String> measurements,
+ List<TSDataType> types,
Review comment:
If user uses wrong type, we will fall at server end. I think we need fall at client to follow the "fall quickly" principle.
##########
File path: service-rpc/rpc-changelist.md
##########
@@ -121,3 +121,35 @@ Last Updated on 2019-10-27 by Lei Rui.
| Add required i64 statementId in TSExecuteStatementReq | Yuan Tian |
| Add required binary time, required list<binary> valueList, required list<binary> bitmapList and remove required binary values, required i32 rowCount in TSQueryDataSet| Yuan Tian |
| Add optional i32 fetchSize in TSExecuteStatementReq,<br />Add optional TSQueryDataSet in TSExecuteStatementResp| liutaohua |
+<<<<<<< HEAD
Review comment:
Sure~
##########
File path: example/rocketmq/src/main/java/org/apache/iotdb/rocketmq/RocketMQConsumer.java
##########
@@ -89,8 +90,37 @@ private void insert(String data) throws IoTDBConnectionException, StatementExecu
String device = dataArray[0];
long time = Long.parseLong(dataArray[1]);
List<String> measurements = Arrays.asList(dataArray[2].split(":"));
- List<String> values = Arrays.asList(dataArray[3].split(":"));
- session.insertRecord(device, time, measurements, values);
+ List<TSDataType> types = new ArrayList<>();
Review comment:
Fixed~
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org