You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2025/07/03 10:58:10 UTC
(iotdb) branch object_type updated (007e7324755 -> 059f3d91d42)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 007e7324755 fix query
new 9890fd67d9e gen file name
new e495ee9ca0b Merge branch 'object_type' of github.com:apache/iotdb into object_type
new 059f3d91d42 merge read part
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../plan/planner/plan/node/write/FileNode.java | 10 ++++++++++
.../node/write/RelationalInsertTabletNode.java | 10 ++++++++++
.../plan/relational/planner/RelationPlanner.java | 12 ++---------
.../unary/scalar/ReadObjectColumnTransformer.java | 2 +-
.../dataregion/memtable/TsFileProcessor.java | 20 +++++++++++++++----
.../org/apache/iotdb/db/utils/ObjectWriter.java | 23 ++++++++++++++--------
6 files changed, 54 insertions(+), 23 deletions(-)
(iotdb) 02/03: Merge branch 'object_type' of github.com:apache/iotdb into object_type
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit e495ee9ca0b32cf3021f31dde4fc8cf21c12e424
Merge: 9890fd67d9e 007e7324755
Author: HTHou <ha...@apache.org>
AuthorDate: Thu Jul 3 18:45:23 2025 +0800
Merge branch 'object_type' of github.com:apache/iotdb into object_type
.../src/main/java/org/apache/iotdb/cli/AbstractCli.java | 1 +
.../java/org/apache/iotdb/isession/SessionDataSet.java | 1 +
.../main/java/org/apache/iotdb/rpc/IoTDBJDBCDataSet.java | 5 +++++
.../main/java/org/apache/iotdb/rpc/IoTDBRpcDataSet.java | 2 ++
.../src/main/java/org/apache/iotdb/session/Session.java | 1 +
.../java/org/apache/iotdb/session/util/SessionUtils.java | 1 +
.../queryengine/execution/aggregation/SumAccumulator.java | 1 +
.../execution/operator/source/SeriesScanUtil.java | 1 +
.../plan/relational/metadata/TableMetadataImpl.java | 3 +--
.../column/unary/scalar/ReadObjectColumnTransformer.java | 14 +++++++-------
.../ReadChunkAlignedSeriesCompactionExecutor.java | 1 +
.../executor/readchunk/SingleSeriesCompactionExecutor.java | 1 +
.../execute/utils/writer/AbstractCompactionWriter.java | 1 +
.../dataregion/read/reader/chunk/MemPageReader.java | 2 ++
.../java/org/apache/iotdb/db/utils/QueryDataSetUtils.java | 2 ++
.../java/org/apache/iotdb/db/utils/TimeValuePairUtils.java | 2 ++
.../org/apache/iotdb/db/utils/datastructure/TVList.java | 4 ++++
17 files changed, 34 insertions(+), 9 deletions(-)
(iotdb) 03/03: merge read part
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 059f3d91d42d27cf57f2a65a2667c78ab328a5a9
Author: HTHou <ha...@apache.org>
AuthorDate: Thu Jul 3 18:57:56 2025 +0800
merge read part
---
.../dag/column/unary/scalar/ReadObjectColumnTransformer.java | 2 +-
.../storageengine/dataregion/memtable/TsFileProcessor.java | 12 ++++++------
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
index 51095e725a3..53de974e895 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/transformation/dag/column/unary/scalar/ReadObjectColumnTransformer.java
@@ -109,6 +109,6 @@ public class ReadObjectColumnTransformer extends UnaryColumnTransformer {
}
private String getObjectPathFromBinary(Binary binary) {
- return "/Users/shuww/Downloads/test_0703/aaa";
+ return binary.toString().split(",")[0];
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index e311e65df7b..af49c77f110 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -572,18 +572,18 @@ public class TsFileProcessor {
String objectFileName =
insertTabletNode.getTimes()[i] + "-" + config.getDataNodeId() + "-" + 1 + ".bin";
File objectFile = new File(writer.getFile().getParent(), objectFileName);
- if (fileNode.isEOF()) {
- relationalInsertTabletNode.getObjectColumn()[i] =
- new Binary(
- (objectFile.getPath() + "," + objectFile.length())
- .getBytes(StandardCharsets.UTF_8));
- }
try (ObjectWriter writer = new ObjectWriter(objectFile)) {
writer.write(fileNode.getContent());
} catch (Exception e) {
throw new WriteProcessException(e);
}
// TODO:[OBJECT] write file node wal
+ if (fileNode.isEOF()) {
+ relationalInsertTabletNode.getObjectColumn()[i] =
+ new Binary(
+ (objectFile.getPath() + "," + objectFile.length())
+ .getBytes(StandardCharsets.UTF_8));
+ }
}
}
}
(iotdb) 01/03: gen file name
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch object_type
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 9890fd67d9e0c4f3c18c3cab21940091d28355d1
Author: HTHou <ha...@apache.org>
AuthorDate: Thu Jul 3 18:44:19 2025 +0800
gen file name
---
.../plan/planner/plan/node/write/FileNode.java | 10 ++++++++++
.../node/write/RelationalInsertTabletNode.java | 10 ++++++++++
.../plan/relational/planner/RelationPlanner.java | 12 ++---------
.../dataregion/memtable/TsFileProcessor.java | 20 +++++++++++++++----
.../org/apache/iotdb/db/utils/ObjectWriter.java | 23 ++++++++++++++--------
5 files changed, 53 insertions(+), 22 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java
index e57d0d7b222..cd82fbf12d8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/FileNode.java
@@ -38,6 +38,16 @@ public class FileNode {
this.content = content;
}
+ public FileNode(boolean isEOF, long offset, byte[] content) {
+ this.isEOF = isEOF;
+ this.offset = offset;
+ this.content = content;
+ }
+
+ public void setFilePath(String filePath) {
+ this.filePath = filePath;
+ }
+
public String getFilePath() {
return filePath;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
index b04aebd0867..40690d012a8 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/RelationalInsertTabletNode.java
@@ -37,6 +37,7 @@ import org.apache.tsfile.enums.TSDataType;
import org.apache.tsfile.file.metadata.IDeviceID;
import org.apache.tsfile.file.metadata.IDeviceID.Factory;
import org.apache.tsfile.read.TimeValuePair;
+import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BitMap;
import org.apache.tsfile.utils.Pair;
import org.apache.tsfile.utils.ReadWriteIOUtils;
@@ -118,6 +119,15 @@ public class RelationalInsertTabletNode extends InsertTabletNode {
return fileNodeList;
}
+ public Binary[] getObjectColumn() {
+ for (int i = 0; i < columns.length; i++) {
+ if (dataTypes[i] == TSDataType.OBJECT) {
+ return (Binary[]) columns[i];
+ }
+ }
+ return null;
+ }
+
@Override
public IDeviceID getDeviceID(int rowIdx) {
if (singleDevice) {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
index 3274de4c9ac..1d6ff3a5c26 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/planner/RelationPlanner.java
@@ -131,7 +131,6 @@ import org.apache.tsfile.utils.Binary;
import org.apache.tsfile.utils.BytesUtils;
import org.apache.tsfile.write.schema.MeasurementSchema;
-import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -1141,16 +1140,9 @@ public class RelationPlanner extends AstVisitor<RelationPlan, Void> {
long offset = BytesUtils.bytesToLong(offsetBytes);
byte[] content = new byte[value.getLength() - 9];
System.arraycopy(value.getValues(), 9, content, 0, value.getLength() - 9);
- // TODO:[OBJECT] Generate File name
- String fileName = "/Users/ht/Documents/iotdb/data/object/file1";
- FileNode fileNode = new FileNode(fileName, isEoF, offset, content);
+ FileNode fileNode = new FileNode(isEoF, offset, content);
fileNodeList.add(fileNode);
- if (isEoF) {
- ((Binary[]) insertTabletStatement.getColumns()[i])[j] =
- new Binary(fileName.getBytes(StandardCharsets.UTF_8));
- } else {
- ((Binary[]) insertTabletStatement.getColumns()[i])[j] = null;
- }
+ ((Binary[]) insertTabletStatement.getColumns()[i])[j] = null;
}
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
index eeddf861b10..e311e65df7b 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/TsFileProcessor.java
@@ -100,6 +100,7 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.nio.channels.ClosedChannelException;
+import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -562,11 +563,22 @@ public class TsFileProcessor {
long[] infoForMetrics)
throws WriteProcessException {
if (insertTabletNode instanceof RelationalInsertTabletNode) {
- List<FileNode> fileNodeList =
- ((RelationalInsertTabletNode) insertTabletNode).getFileNodeList();
+ RelationalInsertTabletNode relationalInsertTabletNode =
+ (RelationalInsertTabletNode) insertTabletNode;
+ List<FileNode> fileNodeList = relationalInsertTabletNode.getFileNodeList();
if (fileNodeList != null) {
- for (FileNode fileNode : fileNodeList) {
- try (ObjectWriter writer = new ObjectWriter(fileNode.getFilePath())) {
+ for (int i = 0; i < fileNodeList.size(); i++) {
+ FileNode fileNode = fileNodeList.get(i);
+ String objectFileName =
+ insertTabletNode.getTimes()[i] + "-" + config.getDataNodeId() + "-" + 1 + ".bin";
+ File objectFile = new File(writer.getFile().getParent(), objectFileName);
+ if (fileNode.isEOF()) {
+ relationalInsertTabletNode.getObjectColumn()[i] =
+ new Binary(
+ (objectFile.getPath() + "," + objectFile.length())
+ .getBytes(StandardCharsets.UTF_8));
+ }
+ try (ObjectWriter writer = new ObjectWriter(objectFile)) {
writer.write(fileNode.getContent());
} catch (Exception e) {
throw new WriteProcessException(e);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java
index 7d572344ce1..070c7b5a149 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/ObjectWriter.java
@@ -19,25 +19,32 @@
package org.apache.iotdb.db.utils;
+import org.apache.commons.io.FileUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.Paths;
public class ObjectWriter implements AutoCloseable {
+ private static final Logger LOGGER = LoggerFactory.getLogger(ObjectWriter.class);
+
private final FileOutputStream fos;
- public ObjectWriter(String filePath) throws FileNotFoundException {
- // TODO:[OBJECT] Dir creation
- Path path = Paths.get(filePath);
- if (!Files.exists(path)) {
+ public ObjectWriter(File filePath) throws FileNotFoundException {
+ try {
+ FileUtils.forceMkdir(filePath.getParentFile());
+ } catch (final IOException e) {
+ throw new FileNotFoundException("Error occurred during creating directory " + filePath);
+ }
+ if (!Files.exists(filePath.toPath())) {
try {
- Files.createFile(path);
+ Files.createFile(filePath.toPath());
} catch (IOException e) {
- e.printStackTrace();
throw new FileNotFoundException(e.getMessage());
}
}