You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ja...@apache.org on 2022/01/28 09:02:32 UTC
[iotdb] branch master updated: Apply string intern for device String (#4964)
This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 6fa97b3 Apply string intern for device String (#4964)
6fa97b3 is described below
commit 6fa97b33a8eb866546e1067e7b225ad2e38487e8
Author: Jianyun Cheng <ch...@360.cn>
AuthorDate: Fri Jan 28 17:00:53 2022 +0800
Apply string intern for device String (#4964)
---
.../db/engine/storagegroup/TsFileResource.java | 2 +-
.../storagegroup/VirtualStorageGroupProcessor.java | 5 ++-
.../storagegroup/timeindex/DeviceTimeIndex.java | 25 +++----------
.../org/apache/iotdb/db/metadata/mnode/MNode.java | 13 +------
.../apache/iotdb/db/rescon/CachedStringPool.java | 42 ----------------------
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
6 files changed, 10 insertions(+), 79 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index a95c075..e4efee2 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -292,7 +292,7 @@ public class TsFileResource {
for (int i = 0; i < size; i++) {
String path = ReadWriteIOUtils.readString(inputStream);
long time = ReadWriteIOUtils.readLong(inputStream);
- deviceMap.put(path, i);
+ deviceMap.put(path.intern(), i);
startTimesArray[i] = time;
}
size = ReadWriteIOUtils.readInt(inputStream);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java
index 19efaf4..039f6e6 100755
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/VirtualStorageGroupProcessor.java
@@ -146,8 +146,7 @@ public class VirtualStorageGroupProcessor {
private static final int MERGE_MOD_START_VERSION_NUM = 1;
private static final Logger logger = LoggerFactory.getLogger(VirtualStorageGroupProcessor.class);
- /** indicating the file to be loaded already exists locally. */
- private static final int POS_ALREADY_EXIST = -2;
+
/** indicating the file to be loaded overlap with some files. */
private static final int POS_OVERLAP = -3;
@@ -554,7 +553,7 @@ public class VirtualStorageGroupProcessor {
Map<String, Long> endTimeMap = new HashMap<>();
for (String deviceId : resource.getDevices()) {
long endTime = resource.getEndTime(deviceId);
- endTimeMap.put(deviceId, endTime);
+ endTimeMap.put(deviceId.intern(), endTime);
}
lastFlushTimeManager.setMultiDeviceLastTime(timePartitionId, endTimeMap);
lastFlushTimeManager.setMultiDeviceFlushedTime(timePartitionId, endTimeMap);
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
index 0859156..b843de0 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/timeindex/DeviceTimeIndex.java
@@ -22,7 +22,6 @@ package org.apache.iotdb.db.engine.storagegroup.timeindex;
import org.apache.iotdb.db.engine.StorageEngine;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.exception.PartitionViolationException;
-import org.apache.iotdb.db.rescon.CachedStringPool;
import org.apache.iotdb.db.utils.SerializeUtils;
import org.apache.iotdb.tsfile.utils.FilePathUtils;
import org.apache.iotdb.tsfile.utils.RamUsageEstimator;
@@ -47,9 +46,6 @@ public class DeviceTimeIndex implements ITimeIndex {
public static final int INIT_ARRAY_SIZE = 64;
- protected static final Map<String, String> cachedDevicePool =
- CachedStringPool.getInstance().getCachedPool();
-
/** start times array. */
protected long[] startTimes;
@@ -92,17 +88,12 @@ public class DeviceTimeIndex implements ITimeIndex {
ReadWriteIOUtils.write(endTimes[i], outputStream);
}
- Map<String, Integer> stringMemoryReducedMap = new ConcurrentHashMap<>();
for (Entry<String, Integer> stringIntegerEntry : deviceToIndex.entrySet()) {
String deviceName = stringIntegerEntry.getKey();
int index = stringIntegerEntry.getValue();
- // To reduce the String number in memory,
- // use the deviceId from cached pool
- stringMemoryReducedMap.put(cachedDevicePool.computeIfAbsent(deviceName, k -> k), index);
ReadWriteIOUtils.write(deviceName, outputStream);
ReadWriteIOUtils.write(index, outputStream);
}
- deviceToIndex = stringMemoryReducedMap;
}
@Override
@@ -120,12 +111,9 @@ public class DeviceTimeIndex implements ITimeIndex {
}
for (int i = 0; i < deviceNum; i++) {
- String path = ReadWriteIOUtils.readString(inputStream);
- // To reduce the String number in memory,
- // use the deviceId from memory instead of the deviceId read from disk
- String cachedPath = cachedDevicePool.computeIfAbsent(path, k -> k);
+ String path = ReadWriteIOUtils.readString(inputStream).intern();
int index = ReadWriteIOUtils.readInt(inputStream);
- deviceToIndex.put(cachedPath, index);
+ deviceToIndex.put(path, index);
}
return this;
}
@@ -144,12 +132,9 @@ public class DeviceTimeIndex implements ITimeIndex {
}
for (int i = 0; i < deviceNum; i++) {
- String path = SerializeUtils.deserializeString(buffer);
- // To reduce the String number in memory,
- // use the deviceId from memory instead of the deviceId read from disk
- String cachedPath = cachedDevicePool.computeIfAbsent(path, k -> k);
+ String path = SerializeUtils.deserializeString(buffer).intern();
int index = buffer.getInt();
- deviceToIndex.put(cachedPath, index);
+ deviceToIndex.put(path, index);
}
return this;
}
@@ -202,7 +187,7 @@ public class DeviceTimeIndex implements ITimeIndex {
index = deviceToIndex.get(deviceId);
} else {
index = deviceToIndex.size();
- deviceToIndex.put(deviceId, index);
+ deviceToIndex.put(deviceId.intern(), index);
if (startTimes.length <= index) {
startTimes = enLargeArray(startTimes, Long.MAX_VALUE);
endTimes = enLargeArray(endTimes, Long.MIN_VALUE);
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
index 839e4e2..1c336ae 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mnode/MNode.java
@@ -21,18 +21,13 @@ package org.apache.iotdb.db.metadata.mnode;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.exception.metadata.IllegalPathException;
import org.apache.iotdb.db.metadata.path.PartialPath;
-import org.apache.iotdb.db.rescon.CachedStringPool;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import java.util.Objects;
public abstract class MNode implements IMNode {
- private static Map<String, String> cachedPathPool =
- CachedStringPool.getInstance().getCachedPool();
-
/** Name of the MNode */
protected String name;
@@ -95,13 +90,7 @@ public abstract class MNode implements IMNode {
@Override
public String getFullPath() {
if (fullPath == null) {
- fullPath = concatFullPath();
- String cachedFullPath = cachedPathPool.get(fullPath);
- if (cachedFullPath == null) {
- cachedPathPool.put(fullPath, fullPath);
- } else {
- fullPath = cachedFullPath;
- }
+ fullPath = concatFullPath().intern();
}
return fullPath;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/CachedStringPool.java b/server/src/main/java/org/apache/iotdb/db/rescon/CachedStringPool.java
deleted file mode 100644
index 84114b3..0000000
--- a/server/src/main/java/org/apache/iotdb/db/rescon/CachedStringPool.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.rescon;
-
-import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
-
-public class CachedStringPool {
-
- private Map<String, String> cachedPool = new ConcurrentHashMap<>();
-
- public Map<String, String> getCachedPool() {
- return cachedPool;
- }
-
- public static CachedStringPool getInstance() {
- return CachedStringPool.InstanceHolder.INSTANCE;
- }
-
- private static class InstanceHolder {
-
- private static final CachedStringPool INSTANCE = new CachedStringPool();
-
- private InstanceHolder() {}
- }
-}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
index cd97a73..b3b25c5 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/TsFileSequenceReader.java
@@ -601,7 +601,7 @@ public class TsFileSequenceReader implements AutoCloseable {
if (metadataIndexNode.getNodeType().equals(MetadataIndexNodeType.LEAF_DEVICE)) {
deviceList.addAll(
metadataIndexNode.getChildren().stream()
- .map(MetadataIndexEntry::getName)
+ .map(x -> x.getName().intern())
.collect(Collectors.toList()));
return deviceList;
}