You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ro...@apache.org on 2024/10/11 12:59:58 UTC
(iotdb) 01/06: refactor TabletInsertionEventParser
This is an automated email from the ASF dual-hosted git repository.
rong pushed a commit to branch container-table-pattern
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit f15d0bf554bf5d2eeb086d4c7b7d5a51707d49fe
Author: Steve Yurong Su <ro...@apache.org>
AuthorDate: Fri Oct 11 18:07:36 2024 +0800
refactor TabletInsertionEventParser
---
.../tablet/PipeInsertNodeTabletInsertionEvent.java | 15 ++++----
.../common/tablet/PipeRawTabletInsertionEvent.java | 9 ++---
.../tablet/parser/TabletInsertionEventParser.java | 42 ++++++++++++++++++++++
.../TabletInsertionEventTreeParser.java} | 18 ++++++----
.../pipe/event/PipeTabletInsertionEventTest.java | 34 +++++++++---------
5 files changed, 84 insertions(+), 34 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
index d58ebffc89a..22fe78c7f7d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeInsertNodeTabletInsertionEvent.java
@@ -26,6 +26,7 @@ import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.datastructure.pattern.TablePattern;
import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
import org.apache.iotdb.commons.pipe.event.PipeInsertionEvent;
+import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreeParser;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
@@ -64,7 +65,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
private final boolean isAligned;
private final boolean isGeneratedByPipe;
- private List<TabletInsertionDataContainer> dataContainers;
+ private List<TabletInsertionEventTreeParser> dataContainers;
private final PartialPath devicePath;
@@ -313,7 +314,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
public Iterable<TabletInsertionEvent> processRowByRow(
final BiConsumer<Row, RowCollector> consumer) {
return initDataContainers().stream()
- .map(tabletInsertionDataContainer -> tabletInsertionDataContainer.processRowByRow(consumer))
+ .map(tabletInsertionEventParser -> tabletInsertionEventParser.processRowByRow(consumer))
.flatMap(Collection::stream)
.collect(Collectors.toList());
}
@@ -322,7 +323,7 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
public Iterable<TabletInsertionEvent> processTablet(
final BiConsumer<Tablet, RowCollector> consumer) {
return initDataContainers().stream()
- .map(tabletInsertionDataContainer -> tabletInsertionDataContainer.processTablet(consumer))
+ .map(tabletInsertionEventParser -> tabletInsertionEventParser.processTablet(consumer))
.flatMap(Collection::stream)
.collect(Collectors.toList());
}
@@ -335,13 +336,13 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
public List<Tablet> convertToTablets() {
return initDataContainers().stream()
- .map(TabletInsertionDataContainer::convertToTablet)
+ .map(TabletInsertionEventTreeParser::convertToTablet)
.collect(Collectors.toList());
}
/////////////////////////// dataContainer ///////////////////////////
- private List<TabletInsertionDataContainer> initDataContainers() {
+ private List<TabletInsertionEventTreeParser> initDataContainers() {
try {
if (dataContainers != null) {
return dataContainers;
@@ -354,12 +355,12 @@ public class PipeInsertNodeTabletInsertionEvent extends PipeInsertionEvent
case INSERT_TABLET:
case RELATIONAL_INSERT_TABLET:
dataContainers.add(
- new TabletInsertionDataContainer(pipeTaskMeta, this, node, treePattern));
+ new TabletInsertionEventTreeParser(pipeTaskMeta, this, node, treePattern));
break;
case INSERT_ROWS:
for (final InsertRowNode insertRowNode : ((InsertRowsNode) node).getInsertRowNodeList()) {
dataContainers.add(
- new TabletInsertionDataContainer(pipeTaskMeta, this, insertRowNode, treePattern));
+ new TabletInsertionEventTreeParser(pipeTaskMeta, this, insertRowNode, treePattern));
}
break;
default:
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
index 364ebca2807..3be8adb5c9d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/PipeRawTabletInsertionEvent.java
@@ -27,6 +27,7 @@ import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
import org.apache.iotdb.commons.pipe.event.EnrichedEvent;
import org.apache.iotdb.commons.pipe.event.PipeInsertionEvent;
import org.apache.iotdb.commons.utils.TestOnly;
+import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreeParser;
import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent;
import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager;
import org.apache.iotdb.db.pipe.resource.memory.PipeMemoryWeightUtil;
@@ -52,7 +53,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
private PipeTabletMemoryBlock allocatedMemoryBlock;
- private TabletInsertionDataContainer dataContainer;
+ private TabletInsertionEventTreeParser dataContainer;
private volatile ProgressIndex overridingProgressIndex;
@@ -267,7 +268,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
final BiConsumer<Row, RowCollector> consumer) {
if (dataContainer == null) {
dataContainer =
- new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, treePattern);
+ new TabletInsertionEventTreeParser(pipeTaskMeta, this, tablet, isAligned, treePattern);
}
return dataContainer.processRowByRow(consumer);
}
@@ -277,7 +278,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
final BiConsumer<Tablet, RowCollector> consumer) {
if (dataContainer == null) {
dataContainer =
- new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, treePattern);
+ new TabletInsertionEventTreeParser(pipeTaskMeta, this, tablet, isAligned, treePattern);
}
return dataContainer.processTablet(consumer);
}
@@ -296,7 +297,7 @@ public class PipeRawTabletInsertionEvent extends PipeInsertionEvent
// if notNullPattern is not "root", we need to convert the tablet
if (dataContainer == null) {
dataContainer =
- new TabletInsertionDataContainer(pipeTaskMeta, this, tablet, isAligned, treePattern);
+ new TabletInsertionEventTreeParser(pipeTaskMeta, this, tablet, isAligned, treePattern);
}
return dataContainer.convertToTablet();
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java
new file mode 100644
index 00000000000..8d9a0ae480a
--- /dev/null
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventParser.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.pipe.event.common.tablet.parser;
+
+import org.apache.iotdb.pipe.api.access.Row;
+import org.apache.iotdb.pipe.api.collector.RowCollector;
+import org.apache.iotdb.pipe.api.event.dml.insertion.TabletInsertionEvent;
+
+import org.apache.tsfile.write.record.Tablet;
+
+import java.util.List;
+import java.util.function.BiConsumer;
+
+public interface TabletInsertionEventParser {
+
+ boolean isAligned();
+
+ void markAsNeedToReport();
+
+ List<TabletInsertionEvent> processRowByRow(final BiConsumer<Row, RowCollector> consumer);
+
+ List<TabletInsertionEvent> processTablet(final BiConsumer<Tablet, RowCollector> consumer);
+
+ Tablet convertToTablet();
+}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreeParser.java
similarity index 98%
rename from iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java
rename to iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreeParser.java
index ce1e4113b91..6067d6a236d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/TabletInsertionDataContainer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/event/common/tablet/parser/TabletInsertionEventTreeParser.java
@@ -17,7 +17,7 @@
* under the License.
*/
-package org.apache.iotdb.db.pipe.event.common.tablet;
+package org.apache.iotdb.db.pipe.event.common.tablet.parser;
import org.apache.iotdb.commons.pipe.agent.task.meta.PipeTaskMeta;
import org.apache.iotdb.commons.pipe.datastructure.pattern.TreePattern;
@@ -58,9 +58,10 @@ import java.util.function.BiConsumer;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
-public class TabletInsertionDataContainer {
+public class TabletInsertionEventTreeParser implements TabletInsertionEventParser {
- private static final Logger LOGGER = LoggerFactory.getLogger(TabletInsertionDataContainer.class);
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(TabletInsertionEventTreeParser.class);
private final PipeTaskMeta pipeTaskMeta; // used to report progress
private final EnrichedEvent
@@ -95,7 +96,7 @@ public class TabletInsertionDataContainer {
}
}
- public TabletInsertionDataContainer(
+ public TabletInsertionEventTreeParser(
final PipeTaskMeta pipeTaskMeta,
final EnrichedEvent sourceEvent,
final InsertNode insertNode,
@@ -113,7 +114,7 @@ public class TabletInsertionDataContainer {
}
}
- public TabletInsertionDataContainer(
+ public TabletInsertionEventTreeParser(
final PipeTaskMeta pipeTaskMeta,
final EnrichedEvent sourceEvent,
final Tablet tablet,
@@ -126,14 +127,16 @@ public class TabletInsertionDataContainer {
}
@TestOnly
- public TabletInsertionDataContainer(final InsertNode insertNode, final TreePattern pattern) {
+ public TabletInsertionEventTreeParser(final InsertNode insertNode, final TreePattern pattern) {
this(null, null, insertNode, pattern);
}
+ @Override
public boolean isAligned() {
return isAligned;
}
+ @Override
public void markAsNeedToReport() {
shouldReport = true;
}
@@ -612,6 +615,7 @@ public class TabletInsertionDataContainer {
//////////////////////////// process ////////////////////////////
+ @Override
public List<TabletInsertionEvent> processRowByRow(final BiConsumer<Row, RowCollector> consumer) {
if (valueColumns.length == 0 || timestampColumn.length == 0) {
return Collections.emptyList();
@@ -636,6 +640,7 @@ public class TabletInsertionDataContainer {
return rowCollector.convertToTabletInsertionEvents(shouldReport);
}
+ @Override
public List<TabletInsertionEvent> processTablet(final BiConsumer<Tablet, RowCollector> consumer) {
final PipeRowCollector rowCollector = new PipeRowCollector(pipeTaskMeta, sourceEvent);
consumer.accept(convertToTablet(), rowCollector);
@@ -644,6 +649,7 @@ public class TabletInsertionDataContainer {
//////////////////////////// convertToTablet ////////////////////////////
+ @Override
public Tablet convertToTablet() {
if (tablet != null) {
return tablet;
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
index b1629623196..1c82bab76cb 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/event/PipeTabletInsertionEventTest.java
@@ -23,7 +23,7 @@ import org.apache.iotdb.commons.exception.IllegalPathException;
import org.apache.iotdb.commons.path.PartialPath;
import org.apache.iotdb.commons.pipe.datastructure.pattern.PrefixTreePattern;
import org.apache.iotdb.db.pipe.event.common.tablet.PipeRawTabletInsertionEvent;
-import org.apache.iotdb.db.pipe.event.common.tablet.TabletInsertionDataContainer;
+import org.apache.iotdb.db.pipe.event.common.tablet.parser.TabletInsertionEventTreeParser;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode;
import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertTabletNode;
@@ -258,15 +258,15 @@ public class PipeTabletInsertionEventTest {
@Test
public void convertToTabletForTest() {
- TabletInsertionDataContainer container1 =
- new TabletInsertionDataContainer(insertRowNode, new PrefixTreePattern(pattern));
+ TabletInsertionEventTreeParser container1 =
+ new TabletInsertionEventTreeParser(insertRowNode, new PrefixTreePattern(pattern));
Tablet tablet1 = container1.convertToTablet();
boolean isAligned1 = container1.isAligned();
Assert.assertEquals(tablet1, tabletForInsertRowNode);
Assert.assertFalse(isAligned1);
- TabletInsertionDataContainer container2 =
- new TabletInsertionDataContainer(insertTabletNode, new PrefixTreePattern(pattern));
+ TabletInsertionEventTreeParser container2 =
+ new TabletInsertionEventTreeParser(insertTabletNode, new PrefixTreePattern(pattern));
Tablet tablet2 = container2.convertToTablet();
boolean isAligned2 = container2.isAligned();
Assert.assertEquals(tablet2, tabletForInsertTabletNode);
@@ -289,15 +289,15 @@ public class PipeTabletInsertionEventTest {
@Test
public void convertToAlignedTabletForTest() {
- TabletInsertionDataContainer container1 =
- new TabletInsertionDataContainer(insertRowNodeAligned, new PrefixTreePattern(pattern));
+ TabletInsertionEventTreeParser container1 =
+ new TabletInsertionEventTreeParser(insertRowNodeAligned, new PrefixTreePattern(pattern));
Tablet tablet1 = container1.convertToTablet();
boolean isAligned1 = container1.isAligned();
Assert.assertEquals(tablet1, tabletForInsertRowNode);
Assert.assertTrue(isAligned1);
- TabletInsertionDataContainer container2 =
- new TabletInsertionDataContainer(insertTabletNodeAligned, new PrefixTreePattern(pattern));
+ TabletInsertionEventTreeParser container2 =
+ new TabletInsertionEventTreeParser(insertTabletNodeAligned, new PrefixTreePattern(pattern));
Tablet tablet2 = container2.convertToTablet();
boolean isAligned2 = container2.isAligned();
Assert.assertEquals(tablet2, tabletForInsertTabletNode);
@@ -320,8 +320,8 @@ public class PipeTabletInsertionEventTest {
@Test
public void convertToTabletWithFilteredRowsForTest() {
- TabletInsertionDataContainer container1 =
- new TabletInsertionDataContainer(
+ TabletInsertionEventTreeParser container1 =
+ new TabletInsertionEventTreeParser(
null,
new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 111L, 113L),
insertRowNode,
@@ -331,8 +331,8 @@ public class PipeTabletInsertionEventTest {
boolean isAligned1 = container1.isAligned();
Assert.assertFalse(isAligned1);
- TabletInsertionDataContainer container2 =
- new TabletInsertionDataContainer(
+ TabletInsertionEventTreeParser container2 =
+ new TabletInsertionEventTreeParser(
null,
new PipeRawTabletInsertionEvent(tabletForInsertRowNode, 110L, 110L),
insertRowNode,
@@ -342,8 +342,8 @@ public class PipeTabletInsertionEventTest {
boolean isAligned2 = container2.isAligned();
Assert.assertFalse(isAligned2);
- TabletInsertionDataContainer container3 =
- new TabletInsertionDataContainer(
+ TabletInsertionEventTreeParser container3 =
+ new TabletInsertionEventTreeParser(
null,
new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, 111L, 113L),
insertTabletNode,
@@ -353,8 +353,8 @@ public class PipeTabletInsertionEventTest {
boolean isAligned3 = container3.isAligned();
Assert.assertFalse(isAligned3);
- TabletInsertionDataContainer container4 =
- new TabletInsertionDataContainer(
+ TabletInsertionEventTreeParser container4 =
+ new TabletInsertionEventTreeParser(
null,
new PipeRawTabletInsertionEvent(tabletForInsertTabletNode, Long.MIN_VALUE, 109L),
insertTabletNode,