You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/11 12:59:58 UTC

(iotdb) 01/06: refactor TabletInsertionEventParser

This is an automated email from the ASF dual-hosted git repository.

rong pushed a commit to branch container-table-pattern
in repository https://gitbox.apache.org/repos/asf/iotdb.git

commit f15d0bf554bf5d2eeb086d4c7b7d5a51707d49fe
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Fri Oct 11 18:07:36 2024 +0800

    refactor TabletInsertionEventParser
---
 .../tablet/PipeInsertNodeTabletInsertionEvent.java | 15 ++++----
 .../common/tablet/PipeRawTabletInsertionEvent.java |  9 ++---
 .../tablet/parser/TabletInsertionEventParser.java  | 42 ++++++++++++++++++++++
 .../TabletInsertionEventTreeParser.java}           | 18 ++++++----
 .../pipe/event/PipeTabletInsertionEventTest.java   | 34 +++++++++---------
 5 files changed, 84 insertions(+), 34 deletions(-)

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
index d58ebffc89a..22fe78c7f7d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
 import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
 import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
 import org.apache.iotdb.commons.pipe.event.PipeInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreeParser;
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
@@ -64,7 +65,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
   private final boolean isAligned;
   private final boolean isGeneratedByPipe;
 
-  private List<TabletInsertionDataContainer> dataContainers;
+  private List<TabletInsertionEventTreeParser> dataContainers;
 
   private final PartialPath devicePath;
 
@@ -313,7 +314,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
   public Iterable<TabletInsertionEvent> processRowByRow(
       final BiConsumer<Row, RowCollector> consumer) {
     return initDataContainers().stream()
-        .map(tabletInsertionDataContainer -> tabletInsertionDataContainer.processRowByRow(consumer))
+        .map(tabletInsertionEventParser -> tabletInsertionEventParser.processRowByRow(consumer))
         .flatMap(Collection::stream)
         .collect(Collectors.toList());
   }
@@ -322,7 +323,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
   public Iterable<TabletInsertionEvent> processTablet(
       final BiConsumer<Tablet, RowCollector> consumer) {
     return initDataContainers().stream()
-        .map(tabletInsertionDataContainer -> tabletInsertionDataContainer.processTablet(consumer))
+        .map(tabletInsertionEventParser -> tabletInsertionEventParser.processTablet(consumer))
         .flatMap(Collection::stream)
         .collect(Collectors.toList());
   }
@@ -335,13 +336,13 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
 
   public List<Tablet> convertToTablets() {
     return initDataContainers().stream()
-        .map(TabletInsertionDataContainer::convertToTablet)
+        .map(TabletInsertionEventTreeParser::convertToTablet)
         .collect(Collectors.toList());
   }
 
   /////////////////////////// dataContainer ///////////////////////////
 
-  private List<TabletInsertionDataContainer> initDataContainers() {
+  private List<TabletInsertionEventTreeParser> initDataContainers() {
     try {
       if (dataContainers != null) {
         return dataContainers;
@@ -354,12 +355,12 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
         case INSERT_TABLET:
         case RELATIONAL_INSERT_TABLET:
           dataContainers.add(
-              new TabletInsertionDataContainer(pipeTaskMeta, this, node, treePattern));
+              new TabletInsertionEventTreeParser(pipeTaskMeta, this, node, treePattern));
           break;
         case INSERT_ROWS:
           for (final InsertRowNode insertRowNode : ((InsertRowsNode) node).getInsertRowNodeList()) {
             dataContainers.add(
-                new TabletInsertionDataContainer(pipeTaskMeta, this, insertRowNode, treePattern));
+                new TabletInsertionEventTreeParser(pipeTaskMeta, this, insertRowNode, treePattern));
           }
           break;
         default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
index 364ebca2807..3be8adb5c9d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
 import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
 import org.apache.iotdb.commons.pipe.event.PipeInsertionEvent;
 import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreeParser;
 import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
 import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
 import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
@@ -52,7 +53,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
 
   private PipeTabletMemoryBlock allocatedMemoryBlock;
 
-  private TabletInsertionDataContainer dataContainer;
+  private TabletInsertionEventTreeParser dataContainer;
 
   private volatile ProgressIndex overridingProgressIndex;
 
@@ -267,7 +268,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
       final BiConsumer<Row, RowCollector> consumer) {
     if (dataContainer == null) {
       dataContainer =
-          new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, treePattern);
+          new TabletInsertionEventTreeParser(pipeTaskMeta, this, tablet, isAligned, treePattern);
     }
     return dataContainer.processRowByRow(consumer);
   }
@@ -277,7 +278,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
       final BiConsumer<Tablet, RowCollector> consumer) {
     if (dataContainer == null) {
       dataContainer =
-          new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, treePattern);
+          new TabletInsertionEventTreeParser(pipeTaskMeta, this, tablet, isAligned, treePattern);
     }
     return dataContainer.processTablet(consumer);
   }
@@ -296,7 +297,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
     // if notNullPattern is not "root", we need to convert the tablet
     if (dataContainer == null) {
       dataContainer =
-          new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, treePattern);
+          new TabletInsertionEventTreeParser(pipeTaskMeta, this, tablet, isAligned, treePattern);
     }
     return dataContainer.convertToTablet();
   }
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java
new file mode 100644
index 00000000000..8d9a0ae480a
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.event.common.tablet.parser;
+
+import org.apache.iotdb.pipe.api.access.Row;
+import org.apache.iotdb.pipe.api.collector.RowCollector;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+import org.apache.tsfile.write.record.Tablet;
+
+import java.util.List;
+import java.util.function.BiConsumer;
+
+public interface TabletInsertionEventParser {
+
+  boolean isAligned();
+
+  void markAsNeedToReport();
+
+  List<TabletInsertionEvent> processRowByRow(final BiConsumer<Row, RowCollector> consumer);
+
+  List<TabletInsertionEvent> processTablet(final BiConsumer<Tablet, RowCollector> consumer);
+
+  Tablet convertToTablet();
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreeParser.java
similarity index 98%
rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java
rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreeParser.java
index ce1e4113b91..6067d6a236d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreeParser.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package org.apache.iotdb.db.pipe.event.common.tablet;
+package org.apache.iotdb.db.pipe.event.common.tablet.parser;
 
 import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
 import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
@@ -58,9 +58,10 @@ import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 import java.util.stream.IntStream;
 
-public class TabletInsertionDataContainer {
+public class TabletInsertionEventTreeParser implements TabletInsertionEventParser {
 
-  private static final Logger LOGGER = LoggerFactory.getLogger(TabletInsertionDataContainer.class);
+  private static final Logger LOGGER =
+      LoggerFactory.getLogger(TabletInsertionEventTreeParser.class);
 
   private final PipeTaskMeta pipeTaskMeta; // used to report progress
   private final EnrichedEvent
@@ -95,7 +96,7 @@ public class TabletInsertionDataContainer {
     }
   }
 
-  public TabletInsertionDataContainer(
+  public TabletInsertionEventTreeParser(
       final PipeTaskMeta pipeTaskMeta,
       final EnrichedEvent sourceEvent,
       final InsertNode insertNode,
@@ -113,7 +114,7 @@ public class TabletInsertionDataContainer {
     }
   }
 
-  public TabletInsertionDataContainer(
+  public TabletInsertionEventTreeParser(
       final PipeTaskMeta pipeTaskMeta,
       final EnrichedEvent sourceEvent,
       final Tablet tablet,
@@ -126,14 +127,16 @@ public class TabletInsertionDataContainer {
   }
 
   @TestOnly
-  public TabletInsertionDataContainer(final InsertNode insertNode, final TreePattern pattern) {
+  public TabletInsertionEventTreeParser(final InsertNode insertNode, final TreePattern pattern) {
     this(null, null, insertNode, pattern);
   }
 
+  @Override
   public boolean isAligned() {
     return isAligned;
   }
 
+  @Override
   public void markAsNeedToReport() {
     shouldReport = true;
   }
@@ -612,6 +615,7 @@ public class TabletInsertionDataContainer {
 
   ////////////////////////////  process  ////////////////////////////
 
+  @Override
   public List<TabletInsertionEvent> processRowByRow(final BiConsumer<Row, RowCollector> consumer) {
     if (valueColumns.length == 0 || timestampColumn.length == 0) {
       return Collections.emptyList();
@@ -636,6 +640,7 @@ public class TabletInsertionDataContainer {
     return rowCollector.convertToTabletInsertionEvents(shouldReport);
   }
 
+  @Override
   public List<TabletInsertionEvent> processTablet(final BiConsumer<Tablet, RowCollector> consumer) {
     final PipeRowCollector rowCollector = new PipeRowCollector(pipeTaskMeta, sourceEvent);
     consumer.accept(convertToTablet(), rowCollector);
@@ -644,6 +649,7 @@ public class TabletInsertionDataContainer {
 
   ////////////////////////////  convertToTablet  ////////////////////////////
 
+  @Override
   public Tablet convertToTablet() {
     if (tablet != null) {
       return tablet;
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
index b1629623196..1c82bab76cb 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
@@ -23,7 +23,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
 import org.apache.iotdb.commons.path.PartialPath;
 import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixTreePattern;
 import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
-import org.apache.iotdb.db.pipe.event.common.tablet.TabletInsertionDataContainer;
+import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreeParser;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
 import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
@@ -258,15 +258,15 @@ public class PipeTabletInsertionEventTest {
 
   @Test
   public void convertToTabletForTest() {
-    TabletInsertionDataContainer container1 =
-        new TabletInsertionDataContainer(insertRowNode, new PrefixTreePattern(pattern));
+    TabletInsertionEventTreeParser container1 =
+        new TabletInsertionEventTreeParser(insertRowNode, new PrefixTreePattern(pattern));
     Tablet tablet1 = container1.convertToTablet();
     boolean isAligned1 = container1.isAligned();
     Assert.assertEquals(tablet1, tabletForInsertRowNode);
     Assert.assertFalse(isAligned1);
 
-    TabletInsertionDataContainer container2 =
-        new TabletInsertionDataContainer(insertTabletNode, new PrefixTreePattern(pattern));
+    TabletInsertionEventTreeParser container2 =
+        new TabletInsertionEventTreeParser(insertTabletNode, new PrefixTreePattern(pattern));
     Tablet tablet2 = container2.convertToTablet();
     boolean isAligned2 = container2.isAligned();
     Assert.assertEquals(tablet2, tabletForInsertTabletNode);
@@ -289,15 +289,15 @@ public class PipeTabletInsertionEventTest {
 
   @Test
   public void convertToAlignedTabletForTest() {
-    TabletInsertionDataContainer container1 =
-        new TabletInsertionDataContainer(insertRowNodeAligned, new PrefixTreePattern(pattern));
+    TabletInsertionEventTreeParser container1 =
+        new TabletInsertionEventTreeParser(insertRowNodeAligned, new PrefixTreePattern(pattern));
     Tablet tablet1 = container1.convertToTablet();
     boolean isAligned1 = container1.isAligned();
     Assert.assertEquals(tablet1, tabletForInsertRowNode);
     Assert.assertTrue(isAligned1);
 
-    TabletInsertionDataContainer container2 =
-        new TabletInsertionDataContainer(insertTabletNodeAligned, new PrefixTreePattern(pattern));
+    TabletInsertionEventTreeParser container2 =
+        new TabletInsertionEventTreeParser(insertTabletNodeAligned, new PrefixTreePattern(pattern));
     Tablet tablet2 = container2.convertToTablet();
     boolean isAligned2 = container2.isAligned();
     Assert.assertEquals(tablet2, tabletForInsertTabletNode);
@@ -320,8 +320,8 @@ public class PipeTabletInsertionEventTest {
 
   @Test
   public void convertToTabletWithFilteredRowsForTest() {
-    TabletInsertionDataContainer container1 =
-        new TabletInsertionDataContainer(
+    TabletInsertionEventTreeParser container1 =
+        new TabletInsertionEventTreeParser(
             null,
             new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 111L, 113L),
             insertRowNode,
@@ -331,8 +331,8 @@ public class PipeTabletInsertionEventTest {
     boolean isAligned1 = container1.isAligned();
     Assert.assertFalse(isAligned1);
 
-    TabletInsertionDataContainer container2 =
-        new TabletInsertionDataContainer(
+    TabletInsertionEventTreeParser container2 =
+        new TabletInsertionEventTreeParser(
             null,
             new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 110L, 110L),
             insertRowNode,
@@ -342,8 +342,8 @@ public class PipeTabletInsertionEventTest {
     boolean isAligned2 = container2.isAligned();
     Assert.assertFalse(isAligned2);
 
-    TabletInsertionDataContainer container3 =
-        new TabletInsertionDataContainer(
+    TabletInsertionEventTreeParser container3 =
+        new TabletInsertionEventTreeParser(
             null,
             new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 111L, 113L),
             insertTabletNode,
@@ -353,8 +353,8 @@ public class PipeTabletInsertionEventTest {
     boolean isAligned3 = container3.isAligned();
     Assert.assertFalse(isAligned3);
 
-    TabletInsertionDataContainer container4 =
-        new TabletInsertionDataContainer(
+    TabletInsertionEventTreeParser container4 =
+        new TabletInsertionEventTreeParser(
             null,
             new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 109L),
             insertTabletNode,