You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by zy...@apache.org on 2022/12/31 08:38:53 UTC
[iotdb] branch master updated: Fix out of bound Segment access in SchemaFile (#8685)
This is an automated email from the ASF dual-hosted git repository.
zyk pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 56a0b4510b Fix out of bound Segment access in SchemaFile (#8685)
56a0b4510b is described below
commit 56a0b4510b4b2da780b09f3f3bc03113ea984411
Author: ZhaoXin <x_...@163.com>
AuthorDate: Sat Dec 31 16:38:47 2022 +0800
Fix out of bound Segment access in SchemaFile (#8685)
---
.../metadata/mtree/store/disk/schemafile/ISchemaPage.java | 2 --
.../mtree/store/disk/schemafile/ISegmentedPage.java | 2 ++
.../metadata/mtree/store/disk/schemafile/InternalPage.java | 13 +++++++------
.../db/metadata/mtree/store/disk/schemafile/SchemaPage.java | 5 -----
.../metadata/mtree/store/disk/schemafile/SegmentedPage.java | 10 ++++++++--
.../mtree/store/disk/schemafile/pagemgr/PageManager.java | 4 ++--
.../apache/iotdb/db/metadata/mtree/traverser/Traverser.java | 4 +++-
7 files changed, 22 insertions(+), 18 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISchemaPage.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISchemaPage.java
index b27f6f68da..b953d1b794 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISchemaPage.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISchemaPage.java
@@ -109,8 +109,6 @@ public interface ISchemaPage {
return new AliasIndexPage(buffer);
}
- boolean isCapableForSize(short size);
-
void syncPageBuffer();
void flushPageToChannel(FileChannel channel) throws IOException;
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISegmentedPage.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISegmentedPage.java
index 7c7392b54b..fb915084e1 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISegmentedPage.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/ISegmentedPage.java
@@ -64,6 +64,8 @@ public interface ISegmentedPage extends ISchemaPage {
short getSpareSize();
+ boolean isCapableForSegSize(short size);
+
short getSegmentSize(short segId) throws SegmentNotFoundException;
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/InternalPage.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/InternalPage.java
index e61bc1b436..46ee795b52 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/InternalPage.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/InternalPage.java
@@ -79,7 +79,7 @@ public class InternalPage extends SchemaPage implements ISegment<Integer, Intege
@Override
public int insertRecord(String key, Integer pointer) throws RecordDuplicatedException {
// TODO: remove debug parameter INTERNAL_SPLIT_VALVE
- if (spareSize < 8 + 4 + key.getBytes().length + INTERNAL_SPLIT_VALVE) {
+ if (spareSize < COMPOUND_POINT_LENGTH + 4 + key.getBytes().length + INTERNAL_SPLIT_VALVE) {
return -1;
}
@@ -267,7 +267,7 @@ public class InternalPage extends SchemaPage implements ISegment<Integer, Intege
this.pageBuffer.position(this.spareOffset);
dstBuffer.put(this.pageBuffer);
- // migrate p1 to p_n-1, offset of each pointer has not to be modified
+ // migrate p1 to p_n-1, offset of each pointer shall not be modified
dstBuffer.position(PAGE_HEADER_SIZE);
this.pageBuffer.position(PAGE_HEADER_SIZE + COMPOUND_POINT_LENGTH);
this.pageBuffer.limit(PAGE_HEADER_SIZE + COMPOUND_POINT_LENGTH * this.memberNum);
@@ -344,6 +344,7 @@ public class InternalPage extends SchemaPage implements ISegment<Integer, Intege
for (int vi = splitPos; vi <= this.memberNum; vi++) {
if (vi == pos + 1) {
// directly points to the new key, do nothing
+ // offset of mPtr always be corrected below, MIN_VALUE as placeholder
mPtr = compoundPointer(pk, Short.MIN_VALUE);
mKey = key;
} else {
@@ -351,8 +352,8 @@ public class InternalPage extends SchemaPage implements ISegment<Integer, Intege
ai = vi > pos ? vi - 1 : vi;
mPtr = getPointerByIndex(ai);
mKey = getKeyByIndex(ai);
- // this.memberNum --;
- this.spareSize -= COMPOUND_POINT_LENGTH + mKey.getBytes().length + 4;
+ // this.spareSize and this.memNumber will always be corrected below during compaction,
+ // therefore unnecessary to count here
}
memberNum++;
@@ -534,8 +535,8 @@ public class InternalPage extends SchemaPage implements ISegment<Integer, Intege
*
* <ul>
* <li>16 bits: reserved
- * <li>32 bits: page index, which indicate segment actually
- * <li>16 bits: key offset, which denote keys in corresponding segment
+ * <li>32 bits: page index, which points to target content
+ * <li>16 bits: key offset, which denotes where key is in this page/segment.
* </ul>
*/
private long compoundPointer(int pageIndex, short offset) {
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SchemaPage.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SchemaPage.java
index 05d4afcac7..acc64bdd29 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SchemaPage.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SchemaPage.java
@@ -57,11 +57,6 @@ public abstract class SchemaPage implements ISchemaPage {
memberNum = ReadWriteIOUtils.readShort(this.pageBuffer);
}
- @Override
- public boolean isCapableForSize(short size) {
- return spareSize >= size;
- }
-
@Override
public void syncPageBuffer() {
this.pageBuffer.limit(this.pageBuffer.capacity());
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SegmentedPage.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SegmentedPage.java
index cde098ccc1..5b63dc7de8 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SegmentedPage.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/SegmentedPage.java
@@ -171,6 +171,11 @@ public class SegmentedPage extends SchemaPage implements ISegmentedPage {
return spareSize;
}
+ @Override
+ public boolean isCapableForSegSize(short size) {
+ return spareSize >= size + SchemaFileConfig.SEG_OFF_DIG;
+ }
+
@Override
public short getSegmentSize(short segId) throws SegmentNotFoundException {
return getSegment(segId).size();
@@ -201,7 +206,7 @@ public class SegmentedPage extends SchemaPage implements ISegmentedPage {
@Override
public long transplantSegment(ISegmentedPage srcPage, short segId, short newSegSize)
throws MetadataException {
- if (!isCapableForSize(newSegSize)) {
+ if (!isCapableForSegSize(newSegSize)) {
throw new SchemaPageOverflowException(pageIndex);
}
if (maxAppendableSegmentSize() < newSegSize) {
@@ -408,7 +413,8 @@ public class SegmentedPage extends SchemaPage implements ISegmentedPage {
}
private short maxAppendableSegmentSize() {
- return (short) (pageBuffer.limit() - 2 * (memberNum + 1) - spareOffset);
+ return (short)
+ (pageBuffer.limit() - SchemaFileConfig.SEG_OFF_DIG * (memberNum + 1) - spareOffset);
}
/**
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/pagemgr/PageManager.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/pagemgr/PageManager.java
index e755a248bd..a107a56787 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/pagemgr/PageManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/store/disk/schemafile/pagemgr/PageManager.java
@@ -493,14 +493,14 @@ public abstract class PageManager implements IPageManager {
protected ISegmentedPage getMinApplSegmentedPageInMem(short size) {
for (Map.Entry<Integer, ISchemaPage> entry : dirtyPages.entrySet()) {
if (entry.getValue().getAsSegmentedPage() != null
- && entry.getValue().isCapableForSize(size)) {
+ && entry.getValue().getAsSegmentedPage().isCapableForSegSize(size)) {
return dirtyPages.get(entry.getKey()).getAsSegmentedPage();
}
}
for (Map.Entry<Integer, ISchemaPage> entry : pageInstCache.entrySet()) {
if (entry.getValue().getAsSegmentedPage() != null
- && entry.getValue().isCapableForSize(size)) {
+ && entry.getValue().getAsSegmentedPage().isCapableForSegSize(size)) {
markDirty(entry.getValue());
return pageInstCache.get(entry.getKey()).getAsSegmentedPage();
}
diff --git a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java
index 47a16bcb21..37df5099dc 100644
--- a/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/mtree/traverser/Traverser.java
@@ -469,7 +469,9 @@ public abstract class Traverser {
// if the node is usingTemplate, the upperTemplate won't be null or the upperTemplateId won't be
// NON_TEMPLATE.
throw new IllegalStateException(
- "There should not be no template mounted on any ancestor of a node usingTemplate.");
+ String.format(
+ "There should be a template mounted on any ancestor of the node [%s] usingTemplate.",
+ node.getFullPath()));
}
public void setTemplateMap(Map<Integer, Template> templateMap) {