[iotdb] branch rel/0.12 updated: [To rel/0.12][IOTDB-2027] Ignore too many WAL BufferOverflow log (#4466)

[qi...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

qiaojialin pushed a commit to branch rel/0.12
in repository https://gitbox.apache.org/repos/asf/iotdb.git


The following commit(s) were added to refs/heads/rel/0.12 by this push:
     new f1cbed5  [To rel/0.12][IOTDB-2027] Ignore too many WAL BufferOverflow log (#4466)
f1cbed5 is described below

commit f1cbed5e439294e345a9de77ece017fde81f6da2
Author: Alan Choo <43...@users.noreply.github.com>
AuthorDate: Thu Nov 25 21:30:42 2021 +0800

    [To rel/0.12][IOTDB-2027] Ignore too many WAL BufferOverflow log (#4466)
---
 .../apache/iotdb/db/qp/physical/PhysicalPlan.java  |  4 ++
 .../db/writelog/node/ExclusiveWriteLogNode.java    |  9 +++-
 .../apache/iotdb/db/writelog/WriteLogNodeTest.java | 49 ++++++++++++++++++++++
 3 files changed, 61 insertions(+), 1 deletion(-)

diff --git a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
index 3b7790d..ca586b5 100644
--- a/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
+++ b/server/src/main/java/org/apache/iotdb/db/qp/physical/PhysicalPlan.java
@@ -63,6 +63,7 @@ import org.slf4j.LoggerFactory;
 
 import java.io.DataOutputStream;
 import java.io.IOException;
+import java.nio.BufferOverflowException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Collections;
@@ -163,6 +164,9 @@ public abstract class PhysicalPlan {
     } catch (UnsupportedOperationException e) {
       // ignore and throw
       throw e;
+    } catch (BufferOverflowException e) {
+      buffer.reset();
+      throw e;
     } catch (Exception e) {
       logger.error(
           "Rollback buffer entry because error occurs when serializing this physical plan.", e);
diff --git a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
index 6f0b94c..e7dc70b 100644
--- a/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
+++ b/server/src/main/java/org/apache/iotdb/db/writelog/node/ExclusiveWriteLogNode.java
@@ -77,6 +77,8 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
 
   private final AtomicBoolean deleted = new AtomicBoolean(false);
 
+  private int bufferOverflowNum = 0;
+
   /**
    * constructor of ExclusiveWriteLogNode.
    *
@@ -123,7 +125,12 @@ public class ExclusiveWriteLogNode implements WriteLogNode, Comparable<Exclusive
     try {
       plan.serialize(logBufferWorking);
     } catch (BufferOverflowException e) {
-      logger.info("WAL BufferOverflow !");
+      bufferOverflowNum++;
+      if (bufferOverflowNum > 200) {
+        logger.info(
+            "WAL bytebuffer overflows too many times. If this occurs frequently, please increase wal_buffer_size.");
+        bufferOverflowNum = 0;
+      }
       sync();
       plan.serialize(logBufferWorking);
     }
diff --git a/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java b/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java
index 2767813..be564ee 100644
--- a/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/writelog/WriteLogNodeTest.java
@@ -321,4 +321,53 @@ public class WriteLogNodeTest {
       MmapUtil.clean((MappedByteBuffer) byteBuffer);
     }
   }
+
+  @Test
+  public void testBufferOverflowAndRewrite() throws IOException, IllegalPathException {
+    String identifier = "root.logTestDevice";
+
+    InsertRowPlan insertPlan =
+        new InsertRowPlan(
+            new PartialPath(identifier),
+            100,
+            new String[] {"s1", "s2", "s3", "s4"},
+            new TSDataType[] {
+              TSDataType.DOUBLE, TSDataType.INT64, TSDataType.TEXT, TSDataType.BOOLEAN
+            },
+            new String[] {"1.0", "15", "str", "false"});
+
+    // get InsertRowPlan byte size
+    ByteBuffer tmpBuffer =
+        ByteBuffer.allocate(IoTDBDescriptor.getInstance().getConfig().getWalBufferSize() / 2);
+    insertPlan.serialize(tmpBuffer);
+    int size = tmpBuffer.position();
+    // allocate buffers
+    ByteBuffer[] byteBuffers = new ByteBuffer[2];
+    byteBuffers[0] = ByteBuffer.allocateDirect(size + 1);
+    byteBuffers[1] = ByteBuffer.allocateDirect(size + 1);
+    WriteLogNode logNode = new ExclusiveWriteLogNode(identifier);
+    logNode.initBuffer(byteBuffers);
+    // write InsertRowPlan to WAL buffer
+    logNode.write(insertPlan);
+    insertPlan.setTime(200);
+    logNode.write(insertPlan);
+
+    logNode.close();
+
+    File walFile =
+        new File(config.getWalDir() + File.separator + identifier + File.separator + "wal1");
+    assertTrue(walFile.exists());
+
+    ILogReader reader = logNode.getLogReader();
+    insertPlan.setTime(100);
+    assertEquals(insertPlan, reader.next());
+    insertPlan.setTime(200);
+    assertEquals(insertPlan, reader.next());
+    reader.close();
+
+    ByteBuffer[] array = logNode.delete();
+    for (ByteBuffer byteBuffer : array) {
+      MmapUtil.clean((MappedByteBuffer) byteBuffer);
+    }
+  }
 }