You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by qi...@apache.org on 2020/07/31 04:15:40 UTC
[incubator-iotdb] branch master updated: set props in
MeasurementSchema to null by default to release memory (#1585)
This is an automated email from the ASF dual-hosted git repository.
qiaojialin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 86405ca set props in MeasurementSchema to null by default to release memory (#1585)
86405ca is described below
commit 86405ca24c2a7ec7066c6d6596a99e6c84b793ba
Author: Jialin Qiao <qj...@mails.tsinghua.edu.cn>
AuthorDate: Fri Jul 31 12:15:28 2020 +0800
set props in MeasurementSchema to null by default to release memory (#1585)
* set props in MeasurementSchema to null by default
---
.../db/engine/querycontext/ReadOnlyMemChunk.java | 4 +---
.../org/apache/iotdb/db/metadata/MManager.java | 3 ++-
.../java/org/apache/iotdb/db/metadata/MTree.java | 4 +++-
.../org/apache/iotdb/db/metadata/mnode/MNode.java | 4 ++--
.../iotdb/db/metadata/mnode/MeasurementMNode.java | 6 ++++--
.../qp/logical/sys/CreateTimeSeriesOperator.java | 6 +++---
.../db/qp/physical/sys/CreateTimeSeriesPlan.java | 6 +++---
.../iotdb/db/qp/strategy/LogicalGenerator.java | 3 ++-
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 1 -
.../iotdb/db/integration/IoTDBSimpleQueryIT.java | 22 ++++++++++++++++++++++
.../org/apache/iotdb/session/IoTDBSessionIT.java | 8 +++++++-
.../tsfile/write/schema/MeasurementSchema.java | 12 ++++++------
.../write/schema/converter/SchemaBuilderTest.java | 6 +++---
13 files changed, 58 insertions(+), 27 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
index aafe607..bd8e02a 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/querycontext/ReadOnlyMemChunk.java
@@ -40,7 +40,6 @@ public class ReadOnlyMemChunk {
private TSEncoding encoding;
private long version;
- Map<String, String> props;
private int floatPrecision = TSFileDescriptor.getInstance().getConfig().getFloatPrecision();
@@ -57,8 +56,7 @@ public class ReadOnlyMemChunk {
this.dataType = dataType;
this.encoding = encoding;
this.version = version;
- this.props = props;
- if (props.containsKey(Encoder.MAX_POINT_NUMBER)) {
+ if (props != null && props.containsKey(Encoder.MAX_POINT_NUMBER)) {
this.floatPrecision = Integer.parseInt(props.get(Encoder.MAX_POINT_NUMBER));
}
tvList.sort();
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 3e0f029..def92ff 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@ -307,10 +307,11 @@ public class MManager {
String[] args = cmd.trim().split(",", -1);
switch (args[0]) {
case MetadataOperationType.CREATE_TIMESERIES:
- Map<String, String> props = new HashMap<>();
+ Map<String, String> props = null;
if (!args[5].isEmpty()) {
String[] keyValues = args[5].split("&");
String[] kv;
+ props = new HashMap<>();
for (String keyValue : keyValues) {
kv = keyValue.split("=");
props.put(kv[0], kv[1]);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
index 0f8acd4..935a767 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MTree.java
@@ -1016,7 +1016,9 @@ public class MTree implements Serializable {
jsonObject.put("DataType", leafMNode.getSchema().getType());
jsonObject.put("Encoding", leafMNode.getSchema().getEncodingType());
jsonObject.put("Compressor", leafMNode.getSchema().getCompressor());
- jsonObject.put("args", leafMNode.getSchema().getProps().toString());
+ if (leafMNode.getSchema().getProps() != null) {
+ jsonObject.put("args", leafMNode.getSchema().getProps().toString());
+ }
jsonObject.put("StorageGroup", storageGroupName);
}
return jsonObject;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
index 002540d..d614b7f 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
@@ -53,8 +53,8 @@ public class MNode implements Serializable {
*/
protected String fullPath;
- transient Map<String, MNode> children;
- transient Map<String, MNode> aliasChildren;
+ transient Map<String, MNode> children = null;
+ transient Map<String, MNode> aliasChildren = null;
protected transient ReadWriteLock lock = new ReentrantReadWriteLock();
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MeasurementMNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MeasurementMNode.java
index b31edc1..645b34a 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MeasurementMNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MeasurementMNode.java
@@ -132,8 +132,10 @@ public class MeasurementMNode extends MNode {
s.append(",").append(schema.getType().ordinal()).append(",");
s.append(schema.getEncodingType().ordinal()).append(",");
s.append(schema.getCompressor().ordinal()).append(",");
- for (Map.Entry<String, String> entry : schema.getProps().entrySet()) {
- s.append(entry.getKey()).append(":").append(entry.getValue()).append(";");
+ if (schema.getProps() != null) {
+ for (Map.Entry<String, String> entry : schema.getProps().entrySet()) {
+ s.append(entry.getKey()).append(":").append(entry.getValue()).append(";");
+ }
}
s.append(",").append(offset).append(",");
s.append(children == null ? "0" : children.size());
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateTimeSeriesOperator.java b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateTimeSeriesOperator.java
index ad78900..2e4f648 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateTimeSeriesOperator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/logical/sys/CreateTimeSeriesOperator.java
@@ -33,9 +33,9 @@ public class CreateTimeSeriesOperator extends RootOperator {
private TSDataType dataType;
private TSEncoding encoding;
private CompressionType compressor;
- private Map<String, String> props;
- private Map<String, String> attributes;
- private Map<String, String> tags;
+ private Map<String, String> props = null;
+ private Map<String, String> attributes = null;
+ private Map<String, String> tags = null;
public CreateTimeSeriesOperator(int tokenIntType) {
super(tokenIntType);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java
index 5d31b22..98b80dd 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/sys/CreateTimeSeriesPlan.java
@@ -40,9 +40,9 @@ public class CreateTimeSeriesPlan extends PhysicalPlan {
private TSEncoding encoding;
private CompressionType compressor;
private String alias;
- private Map<String, String> props;
- private Map<String, String> tags;
- private Map<String, String> attributes;
+ private Map<String, String> props = null;
+ private Map<String, String> tags = null;
+ private Map<String, String> attributes = null;
public CreateTimeSeriesPlan() {
super(false, Operator.OperatorType.CREATE_TIMESERIES);
diff --git a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
index b6db893..6ad6202 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/strategy/LogicalGenerator.java
@@ -1131,13 +1131,14 @@ public class LogicalGenerator extends SqlBaseBaseListener {
createTimeSeriesOperator.setEncoding(TSEncoding.valueOf(encoding));
CompressionType compressor;
List<PropertyContext> properties = ctx.property();
- Map<String, String> props = new HashMap<>(properties.size());
if (ctx.compressor() != null) {
compressor = CompressionType.valueOf(ctx.compressor().getText().toUpperCase());
} else {
compressor = TSFileDescriptor.getInstance().getConfig().getCompressor();
}
+ Map<String, String> props = null;
if (ctx.property(0) != null) {
+ props = new HashMap<>(properties.size());
for (PropertyContext property : properties) {
props.put(property.ID().getText().toLowerCase(),
property.propertyValue().getText().toLowerCase());
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java
index 8ebc43b..b3af62c 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBMetadataFetchIT.java
@@ -394,7 +394,6 @@ public class IoTDBMetadataFetchIT {
+ "\t\t\t\t\t\t\"Encoding\":\"RLE\"\n"
+ "\t\t\t\t\t},\n"
+ "\t\t\t\t\t\"status\":{\n"
- + "\t\t\t\t\t\t\"args\":\"{}\",\n"
+ "\t\t\t\t\t\t\"StorageGroup\":\"root.ln.wf01.wt01\",\n"
+ "\t\t\t\t\t\t\"DataType\":\"BOOLEAN\",\n"
+ "\t\t\t\t\t\t\"Compressor\":\"SNAPPY\",\n"
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
index 42f3f39..2117b59 100644
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
+++ b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBSimpleQueryIT.java
@@ -19,6 +19,10 @@
package org.apache.iotdb.db.integration;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.metadata.mnode.MNode;
+import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.jdbc.IoTDBSQLException;
@@ -44,6 +48,24 @@ public class IoTDBSimpleQueryIT {
}
@Test
+ public void testCreatTimeseries() throws SQLException, ClassNotFoundException, MetadataException {
+ Class.forName(Config.JDBC_DRIVER_NAME);
+ try(Connection connection = DriverManager
+ .getConnection(Config.IOTDB_URL_PREFIX + "127.0.0.1:6667/",
+ "root", "root");
+ Statement statement = connection.createStatement()){
+ statement.setFetchSize(5);
+ statement.execute("SET STORAGE GROUP TO root.sg1");
+ statement.execute("CREATE TIMESERIES root.sg1.d0.s1 WITH DATATYPE=INT32,ENCODING=PLAIN");
+ } catch (SQLException e) {
+ e.printStackTrace();
+ }
+
+ MeasurementMNode mNode = (MeasurementMNode) MManager.getInstance().getNodeByPath("root.sg1.d0.s1");
+ assertNull(mNode.getSchema().getProps());
+ }
+
+ @Test
public void testEmptyDataSet() throws SQLException, ClassNotFoundException {
Class.forName(Config.JDBC_DRIVER_NAME);
try(Connection connection = DriverManager
diff --git a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java
index 7bc3a11..c735cc4 100644
--- a/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java
+++ b/session/src/test/java/org/apache/iotdb/session/IoTDBSessionIT.java
@@ -20,6 +20,7 @@ package org.apache.iotdb.session;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import java.io.File;
@@ -35,6 +36,9 @@ import java.util.List;
import java.util.Map;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.metadata.MetadataException;
+import org.apache.iotdb.db.metadata.MManager;
+import org.apache.iotdb.db.metadata.mnode.MeasurementMNode;
import org.apache.iotdb.db.utils.EnvironmentUtils;
import org.apache.iotdb.jdbc.Config;
import org.apache.iotdb.rpc.BatchExecutionException;
@@ -296,7 +300,7 @@ public class IoTDBSessionIT {
@Test
public void testCreateMultiTimeseries()
- throws IoTDBConnectionException, BatchExecutionException, StatementExecutionException {
+ throws IoTDBConnectionException, BatchExecutionException, StatementExecutionException, MetadataException {
session = new Session("127.0.0.1", 6667, "root", "root");
session.open();
@@ -325,6 +329,8 @@ public class IoTDBSessionIT {
Assert.assertTrue(session.checkTimeseriesExists("root.sg1.d1.s1"));
Assert.assertTrue(session.checkTimeseriesExists("root.sg1.d1.s2"));
+ MeasurementMNode mNode = (MeasurementMNode) MManager.getInstance().getNodeByPath("root.sg1.d1.s1");
+ assertNull(mNode.getSchema().getProps());
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java
index 0ef47ea..ce72206 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/write/schema/MeasurementSchema.java
@@ -49,7 +49,7 @@ public class MeasurementSchema implements Comparable<MeasurementSchema>, Seriali
private TSEncoding encoding;
private TSEncodingBuilder encodingConverter;
private CompressionType compressor;
- private Map<String, String> props = new HashMap<>();
+ private Map<String, String> props = null;
public MeasurementSchema() {
}
@@ -58,7 +58,7 @@ public class MeasurementSchema implements Comparable<MeasurementSchema>, Seriali
this(measurementId, tsDataType,
TSEncoding.valueOf(TSFileDescriptor.getInstance().getConfig().getValueEncoder()),
TSFileDescriptor.getInstance().getConfig().getCompressor(),
- Collections.emptyMap());
+ null);
}
/**
@@ -67,12 +67,12 @@ public class MeasurementSchema implements Comparable<MeasurementSchema>, Seriali
public MeasurementSchema(String measurementId, TSDataType type, TSEncoding encoding) {
this(measurementId, type, encoding,
TSFileDescriptor.getInstance().getConfig().getCompressor(),
- Collections.emptyMap());
+ null);
}
public MeasurementSchema(String measurementId, TSDataType type, TSEncoding encoding,
CompressionType compressionType) {
- this(measurementId, type, encoding, compressionType, Collections.emptyMap());
+ this(measurementId, type, encoding, compressionType, null);
}
/**
@@ -86,7 +86,7 @@ public class MeasurementSchema implements Comparable<MeasurementSchema>, Seriali
this.type = type;
this.measurementId = measurementId;
this.encoding = encoding;
- this.props = props == null ? Collections.emptyMap() : props;
+ this.props = props;
this.compressor = compressionType;
}
@@ -288,7 +288,7 @@ public class MeasurementSchema implements Comparable<MeasurementSchema>, Seriali
public String toString() {
StringContainer sc = new StringContainer("");
sc.addTail("[", measurementId, ",", type.toString(), ",", encoding.toString(), ",",
- props.toString(), ",",
+ props == null ? "" : props.toString(), ",",
compressor.toString());
sc.addTail("]");
return sc.toString();
diff --git a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java
index 1bd534f..09224c2 100644
--- a/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java
+++ b/tsfile/src/test/java/org/apache/iotdb/tsfile/write/schema/converter/SchemaBuilderTest.java
@@ -48,7 +48,7 @@ public class SchemaBuilderTest {
new MeasurementSchema("s5", TSDataType.INT32, TSEncoding.TS_2DIFF, CompressionType.UNCOMPRESSED, null));
Collection<MeasurementSchema> timeseries = schema.getRegisteredTimeseriesMap().values();
- String[] tsDesStrings = { "[s4,DOUBLE,RLE,{max_point_number=3},SNAPPY]", "[s5,INT32,TS_2DIFF,{},UNCOMPRESSED]" };
+ String[] tsDesStrings = { "[s4,DOUBLE,RLE,{max_point_number=3},SNAPPY]", "[s5,INT32,TS_2DIFF,,UNCOMPRESSED]" };
int i = 0;
for (MeasurementSchema desc : timeseries) {
assertEquals(tsDesStrings[i++], desc.toString());
@@ -70,7 +70,7 @@ public class SchemaBuilderTest {
schema.registerDevice("d1", "template1");
Collection<MeasurementSchema> timeseries = schema.getRegisteredTimeseriesMap().values();
- String[] tsDesStrings = { "[s4,DOUBLE,RLE,{max_point_number=3},SNAPPY]", "[s5,INT32,TS_2DIFF,{},UNCOMPRESSED]" };
+ String[] tsDesStrings = { "[s4,DOUBLE,RLE,{max_point_number=3},SNAPPY]", "[s5,INT32,TS_2DIFF,,UNCOMPRESSED]" };
int i = 0;
for (MeasurementSchema desc : timeseries) {
assertEquals(tsDesStrings[i++], desc.toString());
@@ -98,7 +98,7 @@ public class SchemaBuilderTest {
Collection<MeasurementSchema> timeseries = schema.getRegisteredTimeseriesMap().values();
String[] tsDesStrings = { "[s4,DOUBLE,RLE,{max_point_number=3},SNAPPY]",
- "[s5,INT32,TS_2DIFF,{},UNCOMPRESSED]",
+ "[s5,INT32,TS_2DIFF,,UNCOMPRESSED]",
"[s6,INT64,RLE,{max_point_number=3},SNAPPY]"};
int i = 0;
for (MeasurementSchema desc : timeseries) {