You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by hx...@apache.org on 2019/04/30 01:35:00 UTC
[incubator-iotdb] branch refactor_overflow updated: add jamm for
measuring memory;
This is an automated email from the ASF dual-hosted git repository.
hxd pushed a commit to branch refactor_overflow
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
The following commit(s) were added to refs/heads/refactor_overflow by this push:
new 70a3191 add jamm for measuring memory;
70a3191 is described below
commit 70a31911895cb0e86dcdf2e8b75bb8938466d7e3
Author: xiangdong huang <sa...@gmail.com>
AuthorDate: Tue Apr 30 09:34:38 2019 +0800
add jamm for measuring memory;
---
iotdb/iotdb/conf/iotdb-engine.properties | 5 +
iotdb/pom.xml | 6 +
.../org/apache/cassandra/utils/ObjectSizes.java | 167 +++++++++++++++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 15 ++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 3 +
.../iotdb/db/exception/TooManyChunksException.java | 26 ++++
.../reader/overflow/ConciseChunkMetadata.java | 64 ++++++++
.../overflow/ConciseChunkMetadataWithFileName.java | 23 +++
.../overflow/OverflowSeriesMultiFileReader.java | 33 ++++
.../overflow/OverflowSeriesSingleFileReader.java | 77 ++++++++++
10 files changed, 419 insertions(+)
diff --git a/iotdb/iotdb/conf/iotdb-engine.properties b/iotdb/iotdb/conf/iotdb-engine.properties
index facfa3b..5031068 100644
--- a/iotdb/iotdb/conf/iotdb-engine.properties
+++ b/iotdb/iotdb/conf/iotdb-engine.properties
@@ -232,3 +232,8 @@ IP_white_list=0.0.0.0/0
#1. It's more likely to update historical data, please choose "true".
#2. It's more likely not to update historical data or you don't know exactly, please choose "false".
update_historical_data_possibility=false
+
+# the memory limit (Bytes) for each query;
+# by default, it is 5MB
+# if there are too many overflow files, you may need to enlarge it.
+query_memory_limit = 5242880
\ No newline at end of file
diff --git a/iotdb/pom.xml b/iotdb/pom.xml
index 7ff37e9..5552876 100644
--- a/iotdb/pom.xml
+++ b/iotdb/pom.xml
@@ -70,6 +70,12 @@
<artifactId>commons-lang3</artifactId>
<version>${common.lang3.version}</version>
</dependency>
+ <!-- jamm is for measuring the memory cost -->
+ <dependency>
+ <groupId>com.github.jbellis</groupId>
+ <artifactId>jamm</artifactId>
+ <version>0.3.3</version>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/iotdb/src/main/java/org/apache/cassandra/utils/ObjectSizes.java b/iotdb/src/main/java/org/apache/cassandra/utils/ObjectSizes.java
new file mode 100644
index 0000000..1ca3f97
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/cassandra/utils/ObjectSizes.java
@@ -0,0 +1,167 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+/**
+ * this code is copied from Apache Cassandra 3.11
+ */
+package org.apache.cassandra.utils;
+
+import java.nio.ByteBuffer;
+
+import org.github.jamm.MemoryLayoutSpecification;
+import org.github.jamm.MemoryMeter;
+
+/**
+ * A convenience class for wrapping access to MemoryMeter
+ */
+public class ObjectSizes
+{
+ private static final MemoryMeter meter = new MemoryMeter()
+ .omitSharedBufferOverhead()
+ .withGuessing(MemoryMeter.Guess.FALLBACK_UNSAFE)
+ .ignoreKnownSingletons();
+
+ private static final long BUFFER_EMPTY_SIZE = measure(ByteBuffer.wrap(new byte[0]));
+ private static final long STRING_EMPTY_SIZE = measure("");
+
+ /**
+ * Memory a byte array consumes
+ * @param bytes byte array to get memory size
+ * @return heap-size of the array
+ */
+ public static long sizeOfArray(byte[] bytes)
+ {
+ return sizeOfArray(bytes.length, 1);
+ }
+
+ /**
+ * Memory a long array consumes
+ * @param longs byte array to get memory size
+ * @return heap-size of the array
+ */
+ public static long sizeOfArray(long[] longs)
+ {
+ return sizeOfArray(longs.length, 8);
+ }
+
+ /**
+ * Memory an int array consumes
+ * @param ints byte array to get memory size
+ * @return heap-size of the array
+ */
+ public static long sizeOfArray(int[] ints)
+ {
+ return sizeOfArray(ints.length, 4);
+ }
+
+ /**
+ * Memory a reference array consumes
+ * @param length the length of the reference array
+ * @return heap-size of the array
+ */
+ public static long sizeOfReferenceArray(int length)
+ {
+ return sizeOfArray(length, MemoryLayoutSpecification.SPEC.getReferenceSize());
+ }
+
+ /**
+ * Memory a reference array consumes itself only
+ * @param objects the array to size
+ * @return heap-size of the array (excluding memory retained by referenced objects)
+ */
+ public static long sizeOfArray(Object[] objects)
+ {
+ return sizeOfReferenceArray(objects.length);
+ }
+
+ private static long sizeOfArray(int length, long elementSize)
+ {
+ return MemoryLayoutSpecification.sizeOfArray(length, elementSize);
+ }
+
+ /**
+ * Memory a ByteBuffer array consumes.
+ */
+ public static long sizeOnHeapOf(ByteBuffer[] array)
+ {
+ long allElementsSize = 0;
+ for (int i = 0; i < array.length; i++)
+ if (array[i] != null)
+ allElementsSize += sizeOnHeapOf(array[i]);
+
+ return allElementsSize + sizeOfArray(array);
+ }
+
+ public static long sizeOnHeapExcludingData(ByteBuffer[] array)
+ {
+ return BUFFER_EMPTY_SIZE * array.length + sizeOfArray(array);
+ }
+
+ /**
+ * Memory a byte buffer consumes
+ * @param buffer ByteBuffer to calculate in memory size
+ * @return Total in-memory size of the byte buffer
+ */
+ public static long sizeOnHeapOf(ByteBuffer buffer)
+ {
+ if (buffer.isDirect())
+ return BUFFER_EMPTY_SIZE;
+ // if we're only referencing a sub-portion of the ByteBuffer, don't count the array overhead (assume it's slab
+ // allocated, so amortized over all the allocations the overhead is negligible and better to undercount than over)
+ if (buffer.capacity() > buffer.remaining())
+ return buffer.remaining();
+ return BUFFER_EMPTY_SIZE + sizeOfArray(buffer.capacity(), 1);
+ }
+
+ public static long sizeOnHeapExcludingData(ByteBuffer buffer)
+ {
+ return BUFFER_EMPTY_SIZE;
+ }
+
+ /**
+ * Memory a String consumes
+ * @param str String to calculate memory size of
+ * @return Total in-memory size of the String
+ */
+ public static long sizeOf(String str)
+ {
+ return STRING_EMPTY_SIZE + sizeOfArray(str.length(), 2);
+ }
+
+ /**
+ * @param pojo the object to measure
+ * @return the size on the heap of the instance and all retained heap referenced by it, excluding portions of
+ * ByteBuffer that are not directly referenced by it but including any other referenced that may also be retained
+ * by other objects.
+ */
+ public static long measureDeep(Object pojo)
+ {
+ return meter.measureDeep(pojo);
+ }
+
+ /**
+ * @param pojo the object to measure
+ * @return the size on the heap of the instance only, excluding any referenced objects
+ */
+ public static long measure(Object pojo)
+ {
+ return meter.measure(pojo);
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index f588430..6509812 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -277,6 +277,13 @@ public class IoTDBConfig {
*/
private long cacheFileReaderClearPeriod = 100000;
+ /**
+ * the memory limit (Bytes) for each query;
+ * by default, it is 5MB
+ * if there are too many overflow files, you may need to enlarge it.
+ */
+ private long queryMemoryLimit = 5242880;
+
public IoTDBConfig() {
// empty constructor
}
@@ -826,4 +833,12 @@ public class IoTDBConfig {
public void setCacheFileReaderClearPeriod(long cacheFileReaderClearPeriod) {
this.cacheFileReaderClearPeriod = cacheFileReaderClearPeriod;
}
+
+ public long getQueryMemoryLimit() {
+ return queryMemoryLimit;
+ }
+
+ public void setQueryMemoryLimit(long queryMemoryLimit) {
+ this.queryMemoryLimit = queryMemoryLimit;
+ }
}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index a665849..57cddf7 100644
--- a/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -252,6 +252,9 @@ public class IoTDBDescriptor {
conf.getZoneID(), e);
}
+ conf.setQueryMemoryLimit(Long.parseLong(properties.getProperty("query_memory_limit",
+ Long.toString(conf.getQueryMemoryLimit()))));
+
} catch (IOException e) {
LOGGER.warn("Cannot load config file because, use default configuration", e);
} catch (Exception e) {
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/exception/TooManyChunksException.java b/iotdb/src/main/java/org/apache/iotdb/db/exception/TooManyChunksException.java
new file mode 100644
index 0000000..c0062f1
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/exception/TooManyChunksException.java
@@ -0,0 +1,26 @@
+package org.apache.iotdb.db.exception;
+
+import java.io.File;
+import java.util.Arrays;
+
+/**
+ * once a TooManyChunksException is thrown, either enlarge the memory resource for queries, or begin
+ * to merge the files in this exception.
+ */
+public class TooManyChunksException extends Exception {
+
+ File[] files;
+
+ public TooManyChunksException(File[] files, String device, String measurment,
+ long numberOfWays) {
+ super(String.format(
+ "Files (%s) has too many Chunks for %s' %s, while query is only allocated %d Chunk spaces. Pls merge the OF file frist before you use it",
+ Arrays.toString(files), device, measurment, numberOfWays));
+ this.files = files;
+ }
+
+ public File[] getFiles() {
+ return files;
+ }
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/ConciseChunkMetadata.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/ConciseChunkMetadata.java
new file mode 100644
index 0000000..66250f0
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/ConciseChunkMetadata.java
@@ -0,0 +1,64 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.reader.overflow;
+
+import org.apache.cassandra.utils.ObjectSizes;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+
+public class ConciseChunkMetadata implements Comparable<ConciseChunkMetadata>{
+
+ private static long EMPTY_SIZE = ObjectSizes.measure(new ConciseChunkMetadata(0,0));
+
+ long startTime;
+ /**
+ * @see org.apache.iotdb.tsfile.file.metadata.ChunkMetaData#getOffsetOfChunkHeader()
+ */
+ long fileOffset;
+
+ public ConciseChunkMetadata(long startTime, long fileOffset) {
+ this.startTime = startTime;
+ this.fileOffset = fileOffset;
+ }
+
+ public ConciseChunkMetadata(ChunkMetaData metaData) {
+ this(metaData.getStartTime(), metaData.getOffsetOfChunkHeader());
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ /**
+ * @see org.apache.iotdb.tsfile.file.metadata.ChunkMetaData#getOffsetOfChunkHeader()
+ */
+ public long getFileOffset() {
+ return fileOffset;
+ }
+
+
+ @Override
+ public int compareTo(ConciseChunkMetadata o) {
+ return Long.compare(startTime, o.startTime);
+ }
+
+ public static long getOccupiedMemory() {
+ return EMPTY_SIZE;
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/ConciseChunkMetadataWithFileName.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/ConciseChunkMetadataWithFileName.java
new file mode 100644
index 0000000..f1130dc
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/ConciseChunkMetadataWithFileName.java
@@ -0,0 +1,23 @@
+package org.apache.iotdb.db.query.reader.overflow;
+
+import java.io.File;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+
+public class ConciseChunkMetadataWithFileName extends ConciseChunkMetadata {
+
+ File file;
+
+ public ConciseChunkMetadataWithFileName(long startTime, long fileOffset, File file) {
+ super(startTime, fileOffset);
+ this.file = file;
+ }
+
+ public ConciseChunkMetadataWithFileName(ChunkMetaData metaData, File file) {
+ super(metaData);
+ this.file = file;
+ }
+
+ public File getFile() {
+ return file;
+ }
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesMultiFileReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesMultiFileReader.java
new file mode 100644
index 0000000..2b6b3bf
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesMultiFileReader.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.reader.overflow;
+
+import java.util.LinkedList;
+import java.util.List;
+
+public class OverflowSeriesMultiFileReader {
+ private String device;
+ private String measurement;
+
+ //this list should be sorted.
+ private List<ConciseChunkMetadataWithFileName> metadataList = new LinkedList<>();
+
+
+}
diff --git a/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesSingleFileReader.java b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesSingleFileReader.java
new file mode 100644
index 0000000..6ecbb8f
--- /dev/null
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/reader/overflow/OverflowSeriesSingleFileReader.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with the License. You may obtain
+ * a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.query.reader.overflow;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import org.apache.iotdb.db.exception.TooManyChunksException;
+import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
+import org.apache.iotdb.tsfile.file.metadata.ChunkGroupMetaData;
+import org.apache.iotdb.tsfile.file.metadata.ChunkMetaData;
+import org.apache.iotdb.tsfile.file.metadata.TsDeviceMetadata;
+import org.apache.iotdb.tsfile.file.metadata.TsFileMetaData;
+import org.apache.iotdb.tsfile.read.TsFileSequenceReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class OverflowSeriesSingleFileReader {
+
+ private static final Logger LOGGER = LoggerFactory
+ .getLogger(OverflowSeriesSingleFileReader.class);
+ private String device;
+ private String measurement;
+ private File file;
+
+ //how many chunks can be read at the same time.
+ private int numberOfWays;
+
+ //this list should be sorted.
+ private List<ConciseChunkMetadata> metadataList = new LinkedList<>();
+
+ public OverflowSeriesSingleFileReader(String device, String measurement, File file,
+ long memoryLimitInBytes) throws IOException, TooManyChunksException {
+ this.device = device;
+ this.measurement = measurement;
+ this.file = file;
+ this.numberOfWays =
+ (int) memoryLimitInBytes / (TSFileConfig.pageSizeInByte + (int) ConciseChunkMetadata
+ .getOccupiedMemory());
+
+ try (TsFileSequenceReader reader = new TsFileSequenceReader(file.getAbsolutePath(), true)) {
+ TsFileMetaData fileMetaData = reader.readFileMetadata();
+ if (fileMetaData.getMeasurementSchema().containsKey(measurement)) {
+ TsDeviceMetadata metadata = reader
+ .readTsDeviceMetaData(fileMetaData.getDeviceMetadataIndex(device));
+ for (ChunkGroupMetaData cgMetadata : metadata.getChunkGroupMetaDataList()) {
+ for (ChunkMetaData metaData : cgMetadata.getChunkMetaDataList()) {
+ if (metaData.getMeasurementUid().equals(measurement)) {
+ this.metadataList.add(new ConciseChunkMetadata(metaData));
+ if (metadataList.size() > numberOfWays) {
+ throw new TooManyChunksException(new File[]{file}, device, measurement,
+ numberOfWays);
+ }
+ break;
+ }
+ }
+ }
+ }
+ Collections.sort(this.metadataList);
+ }
+ }
+}