[GitHub] [iotdb] SilverNarcissus commented on a change in pull request #1524: [IOTDB-776] Control the memory usage of flushing the memtable

[GitBox <gi...@apache.org>]

SilverNarcissus commented on a change in pull request #1524:
URL: https://github.com/apache/iotdb/pull/1524#discussion_r511765603



##########
File path: server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileProcessorInfo.java
##########
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.engine.storagegroup;
+
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+
+/**
+ * The TsFileProcessorInfo records the memory cost of this TsFileProcessor.
+ */
+public class TsFileProcessorInfo {
+
+  /**
+   * Once tspInfo updated, report to storageGroupInfo that this TSP belongs to.
+   */
+  private StorageGroupInfo storageGroupInfo;
+
+  /**
+   * The memory cost of the unsealed TsFileResources of this TSP
+   */
+  private long unsealedResourceMemCost;
+  
+  /**
+   * The memory cost of memTable of this TSP
+   */
+  private long memTableCost;
+
+  /**
+   * The memory cost of ChunkMetadata of this TSP
+   */
+  private long chunkMetadataMemCost;
+
+  /**
+   * The memory cost of WAL of this TSP
+   */
+  private long walMemCost;
+
+  public TsFileProcessorInfo(StorageGroupInfo storageGroupInfo) {
+    this.storageGroupInfo = storageGroupInfo;
+    this.unsealedResourceMemCost = 0;
+    this.memTableCost = 0;
+    this.chunkMetadataMemCost = 0;
+    this.walMemCost = IoTDBDescriptor.getInstance().getConfig().getWalBufferSize();
+  }
+
+  public void addUnsealedResourceMemCost(long cost) {
+    unsealedResourceMemCost += cost;
+    storageGroupInfo.addUnsealedResourceMemCost(cost);
+  }
+
+  public void addChunkMetadataMemCost(long cost) {
+    chunkMetadataMemCost += cost;
+    storageGroupInfo.addChunkMetadataMemCost(cost);
+  }
+
+  public void addMemTableCost(long cost) {
+    memTableCost += cost;
+    storageGroupInfo.addMemTableCost(cost);
+  }
+
+  /**
+   * call this method when closing TSP
+   */
+  public void clear() {
+    storageGroupInfo.resetUnsealedResourceMemCost(unsealedResourceMemCost);
+    storageGroupInfo.resetChunkMetadataMemCost(chunkMetadataMemCost);
+    storageGroupInfo.resetWalMemCost(walMemCost);

Review comment:
       Have you fixed the concurrent issue? I think current implementation isn't thread safe

##########
File path: server/src/main/java/org/apache/iotdb/db/rescon/PrimitiveArrayManager.java
##########
@@ -0,0 +1,329 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iotdb.db.rescon;
+
+import java.util.ArrayDeque;
+import java.util.Arrays;
+import java.util.EnumMap;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+import org.apache.iotdb.db.conf.IoTDBConfig;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Binary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Manage all primitive data list in memory, including get and release operation.
+ */
+public class PrimitiveArrayManager {
+
+  /**
+   * data type -> ArrayDeque<Array>
+   */
+  private static final Map<TSDataType, ArrayDeque<Object>> bufferedArraysMap = new EnumMap<>(
+      TSDataType.class);
+
+  /**
+   * data type -> current number of buffered arrays
+   */
+  private static final Map<TSDataType, Integer> bufferedArraysNumMap = new EnumMap<>(
+      TSDataType.class);
+
+  /**
+   * data type -> ratio of data type in schema, which could be seen as recommended ratio
+   */
+  private static final Map<TSDataType, Double> bufferedArraysNumRatio = new EnumMap<>(
+      TSDataType.class);
+
+  private static final Logger logger = LoggerFactory.getLogger(PrimitiveArrayManager.class);
+
+  private static final IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
+
+  public static final int ARRAY_SIZE = config.getPrimitiveArraySize();
+
+  /**
+   * threshold total size of arrays for all data types
+   */
+  private static final double BUFFERED_ARRAY_SIZE_THRESHOLD =
+      config.getAllocateMemoryForWrite() * config.getBufferedArraysMemoryProportion();
+
+  /**
+   * total size of buffered arrays
+   */
+  private static AtomicInteger bufferedArraysSize = new AtomicInteger();
+
+  /**
+   * total size of out of buffer arrays
+   */
+  private static AtomicInteger outOfBufferArraysSize = new AtomicInteger();
+
+  static {
+    bufferedArraysMap.put(TSDataType.BOOLEAN, new ArrayDeque<>());
+    bufferedArraysMap.put(TSDataType.INT32, new ArrayDeque<>());
+    bufferedArraysMap.put(TSDataType.INT64, new ArrayDeque<>());
+    bufferedArraysMap.put(TSDataType.FLOAT, new ArrayDeque<>());
+    bufferedArraysMap.put(TSDataType.DOUBLE, new ArrayDeque<>());
+    bufferedArraysMap.put(TSDataType.TEXT, new ArrayDeque<>());
+  }
+
+  private PrimitiveArrayManager() {
+  }
+
+  /**
+   * Get primitive data lists according to type
+   *
+   * @param dataType data type
+   * @return an array
+   */
+  public static Object getDataListByType(TSDataType dataType) {
+    // check buffered array num
+    if (bufferedArraysSize.get() + ARRAY_SIZE * dataType.getDataTypeSize()
+        > BUFFERED_ARRAY_SIZE_THRESHOLD) {
+      // return an out of buffer array
+      outOfBufferArraysSize.addAndGet(ARRAY_SIZE * dataType.getDataTypeSize());
+      return getDataList(dataType);
+    }
+
+    synchronized (bufferedArraysMap.get(dataType)) {
+      // return a buffered array
+      bufferedArraysNumMap.put(dataType, bufferedArraysNumMap.getOrDefault(dataType, 0) + 1);
+      bufferedArraysSize.addAndGet(ARRAY_SIZE * dataType.getDataTypeSize());
+      Object dataArray = bufferedArraysMap.get(dataType).poll();
+      if (dataArray != null) {
+        return dataArray;
+      }
+    }
+    return getDataList(dataType);

Review comment:
       We may not get here, so we should return null and add some comments

##########
File path: tsfile/src/main/java/org/apache/iotdb/tsfile/file/metadata/enums/TSDataType.java
##########
@@ -98,6 +99,23 @@ public short serialize() {
     return enumToByte();
   }
 
+  public int getDataTypeSize() {
+    switch (this) {
+      case BOOLEAN:
+        return 1;
+      case INT32:
+      case FLOAT:
+        // For text: return the size of reference here

Review comment:
       Reference's length is 8 bytes. Or 16 bytes exactly (Java objects head length)

##########
File path: server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
##########
@@ -608,13 +644,28 @@ public void deleteStorageGroups(List<PartialPath> storageGroups) throws Metadata
           logWriter.deleteStorageGroup(storageGroup.getFullPath());
         }
       }
-    } catch (ConfigAdjusterException e) {
-      throw new MetadataException(e);
     } catch (IOException e) {
       throw new MetadataException(e.getMessage());
     }
   }
 
+  /**
+   * update statistics in schemaDataTypeNumMap
+   *
+   * @param type data type
+   * @param num 1 for creating timeseries and -1 for deleting timeseries
+   */
+  private void updateSchemaDataTypeNumMap(TSDataType type, int num) {
+    schemaDataTypeNumMap.put(type, schemaDataTypeNumMap.getOrDefault(type, 0) + num);
+    schemaDataTypeNumMap.put(TSDataType.INT64,
+        schemaDataTypeNumMap.getOrDefault(TSDataType.INT64, 0) + num);
+    int currentDataTypeTotalNum = schemaDataTypeNumMap.values().size();
+    if (num > 0 && currentDataTypeTotalNum >= reportedDataTypeTotalNum * 1.1) {

Review comment:
       "1.1" should be a static final constant and have a descriptive name




----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

For queries about this service, please contact Infrastructure at:
users@infra.apache.org