You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by lz...@apache.org on 2022/09/13 09:44:06 UTC
[flink-table-store] branch release-0.2 updated: [FLINK-29273] Page not enough Exception in SortBufferMemTable
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.2
in repository https://gitbox.apache.org/repos/asf/flink-table-store.git
The following commit(s) were added to refs/heads/release-0.2 by this push:
new f33a12be [FLINK-29273] Page not enough Exception in SortBufferMemTable
f33a12be is described below
commit f33a12be878006a915daab2ac6e0e0d5e484486d
Author: Jingsong Lee <ji...@gmail.com>
AuthorDate: Tue Sep 13 17:43:26 2022 +0800
[FLINK-29273] Page not enough Exception in SortBufferMemTable
This closes #291
---
.../table/store/file/memory/MemoryPoolFactory.java | 22 ++++++--
.../store/file/memory/MemoryPoolFactoryTest.java | 59 ++++++++++++++++++++++
.../table/store/table/WritePreemptMemoryTest.java | 5 +-
3 files changed, 81 insertions(+), 5 deletions(-)
diff --git a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/memory/MemoryPoolFactory.java b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/memory/MemoryPoolFactory.java
index 2e06a9ec..ef1583d5 100644
--- a/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/memory/MemoryPoolFactory.java
+++ b/flink-table-store-core/src/main/java/org/apache/flink/table/store/file/memory/MemoryPoolFactory.java
@@ -30,16 +30,21 @@ import java.util.List;
public class MemoryPoolFactory {
private final MemorySegmentPool innerPool;
+ private final int totalPages;
private final Iterable<MemoryOwner> owners;
public MemoryPoolFactory(MemorySegmentPool innerPool, Iterable<MemoryOwner> owners) {
this.innerPool = innerPool;
+ this.totalPages = innerPool.freePages();
this.owners = owners;
}
public void notifyNewOwner(MemoryOwner owner) {
- MemorySegmentPool memoryPool = new OwnerMemoryPool(owner);
- owner.setMemoryPool(memoryPool);
+ owner.setMemoryPool(createSubPool(owner));
+ }
+
+ MemorySegmentPool createSubPool(MemoryOwner owner) {
+ return new OwnerMemoryPool(owner);
}
private void preemptMemory(MemoryOwner owner) {
@@ -67,6 +72,8 @@ public class MemoryPoolFactory {
private final MemoryOwner owner;
+ private int allocatedPages = 0;
+
public OwnerMemoryPool(MemoryOwner owner) {
this.owner = owner;
}
@@ -78,12 +85,16 @@ public class MemoryPoolFactory {
@Override
public void returnAll(List<MemorySegment> memory) {
+ allocatedPages -= memory.size();
innerPool.returnAll(memory);
}
@Override
public int freePages() {
- return innerPool.freePages();
+ // Actually, other owners still keep 1 page
+ // TODO We need to optimize this one page later.
+ // See BinaryInMemorySortBuffer.reset
+ return totalPages - allocatedPages;
}
@Override
@@ -91,7 +102,10 @@ public class MemoryPoolFactory {
MemorySegment segment = innerPool.nextSegment();
if (segment == null) {
preemptMemory(owner);
- return innerPool.nextSegment();
+ segment = innerPool.nextSegment();
+ }
+ if (segment != null) {
+ allocatedPages++;
}
return segment;
}
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/memory/MemoryPoolFactoryTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/memory/MemoryPoolFactoryTest.java
new file mode 100644
index 00000000..891fdad6
--- /dev/null
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/file/memory/MemoryPoolFactoryTest.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.table.store.file.memory;
+
+import org.apache.flink.table.runtime.util.MemorySegmentPool;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Test for {@link MemoryPoolFactory}. */
+public class MemoryPoolFactoryTest {
+
+ @Test
+ public void testFreePages() {
+ MemoryPoolFactory factory =
+ new MemoryPoolFactory(
+ new HeapMemorySegmentPool(1024 * 10, 1024), new ArrayList<>());
+ MemorySegmentPool pool1 = factory.createSubPool(new TestMemoryOwner());
+ MemorySegmentPool pool2 = factory.createSubPool(new TestMemoryOwner());
+ assertThat(pool1.nextSegment()).isNotNull();
+ assertThat(pool2.nextSegment()).isNotNull();
+ assertThat(pool2.nextSegment()).isNotNull();
+
+ assertThat(pool1.freePages()).isEqualTo(9);
+ assertThat(pool2.freePages()).isEqualTo(8);
+ }
+
+ private static class TestMemoryOwner implements MemoryOwner {
+ @Override
+ public void setMemoryPool(MemorySegmentPool memoryPool) {}
+
+ @Override
+ public long memoryOccupancy() {
+ return 0;
+ }
+
+ @Override
+ public void flushMemory() {}
+ }
+}
diff --git a/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/WritePreemptMemoryTest.java b/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/WritePreemptMemoryTest.java
index 326de0dc..7ddb76dd 100644
--- a/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/WritePreemptMemoryTest.java
+++ b/flink-table-store-core/src/test/java/org/apache/flink/table/store/table/WritePreemptMemoryTest.java
@@ -93,7 +93,10 @@ public class WritePreemptMemoryTest extends FileStoreTableTestBase {
Configuration conf = new Configuration();
conf.set(CoreOptions.PATH, tablePath.toString());
conf.set(CoreOptions.WRITE_MODE, WriteMode.CHANGE_LOG);
- conf.set(CoreOptions.WRITE_BUFFER_SIZE, new MemorySize(30 * 1024));
+ // Run with minimal memory to ensure a more intense preempt
+ // Currently a writer needs at least one page
+ int pages = 10;
+ conf.set(CoreOptions.WRITE_BUFFER_SIZE, new MemorySize(pages * 1024));
conf.set(CoreOptions.PAGE_SIZE, new MemorySize(1024));
configure.accept(conf);
SchemaManager schemaManager = new SchemaManager(tablePath);