You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ha...@apache.org on 2024/04/19 07:46:50 UTC
(iotdb) branch rc/1.3.1 updated (542ccdec602 -> 040ab6505de)
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a change to branch rc/1.3.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
from 542ccdec602 Pipe: Disable unstable features in the distribution
new 711b83c1b91 delay estimate memory of InnerSpaceCompactionTask (#12314)
new a06ff596894 Do not set read only in compaction module (#12311)
new 040ab6505de fix(jdbc): when parameter is string ,append "\"" (#12371)
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 7 ++-
.../iotdb/jdbc/IoTDBPreparedStatementTest.java | 23 ++++++++++
.../exception/CompactionExceptionHandler.java | 15 +++----
.../execute/recover/CompactionRecoverTask.java | 18 ++------
.../execute/task/AbstractCompactionTask.java | 10 ++---
.../execute/task/InnerSpaceCompactionTask.java | 2 +-
.../compaction/schedule/CompactionScheduler.java | 6 ---
.../compaction/schedule/CompactionTaskQueue.java | 52 ++++++++++------------
.../impl/RewriteCrossSpaceCompactionSelector.java | 2 +-
.../inner/InnerSpaceCompactionExceptionTest.java | 5 +--
10 files changed, 70 insertions(+), 70 deletions(-)
(iotdb) 01/03: delay estimate memory of InnerSpaceCompactionTask (#12314)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 711b83c1b91e48f3461296944760a7c7bf47e3e8
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Wed Apr 10 20:36:26 2024 +0800
delay estimate memory of InnerSpaceCompactionTask (#12314)
---
.../compaction/schedule/CompactionScheduler.java | 6 ---
.../compaction/schedule/CompactionTaskQueue.java | 52 ++++++++++------------
2 files changed, 24 insertions(+), 34 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
index 707f251ded7..6ebb5578670 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduler.java
@@ -200,12 +200,6 @@ public class CompactionScheduler {
"Compaction task start check failed because disk free ratio is less than disk_space_warning_threshold");
return false;
}
- // check task memory cost
- long allocatedTotalCompactionMemory = SystemInfo.getInstance().getMemorySizeForCompaction();
- long estimatedTaskMemoryCost = task.getEstimatedMemoryCost();
- if (estimatedTaskMemoryCost < 0 || estimatedTaskMemoryCost > allocatedTotalCompactionMemory) {
- return false;
- }
return true;
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskQueue.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskQueue.java
index 66a6900afb1..9f0a5dbbc7d 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskQueue.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionTaskQueue.java
@@ -36,48 +36,44 @@ public class CompactionTaskQueue extends FixedPriorityBlockingQueue<AbstractComp
public AbstractCompactionTask take() throws InterruptedException {
final ReentrantLock lock = this.lock;
while (true) {
+ AbstractCompactionTask task = null;
lock.lockInterruptibly();
try {
while (queue.isEmpty()) {
notEmpty.await();
}
- AbstractCompactionTask task = tryPollExecutableTask();
- // task == null indicates that there is no runnable task now
- if (task != null) {
- return task;
- }
+ task = queue.pollFirst();
} finally {
lock.unlock();
}
- Thread.sleep(TimeUnit.SECONDS.toMillis(1));
- }
- }
-
- private AbstractCompactionTask tryPollExecutableTask() {
- while (true) {
- if (queue.isEmpty()) {
- return null;
- }
- AbstractCompactionTask task = queue.pollFirst();
- if (task == null) {
- continue;
- }
- if (!checkTaskValid(task)) {
- dropCompactionTask(task);
- continue;
- }
- if (!task.tryOccupyResourcesForRunning()) {
- queue.add(task);
- return null;
- }
- if (!transitTaskFileStatus(task)) {
- dropCompactionTask(task);
+ boolean prepareTaskSuccess = prepareTask(task);
+ if (!prepareTaskSuccess) {
+ Thread.sleep(TimeUnit.SECONDS.toMillis(1));
continue;
}
return task;
}
}
+ private boolean prepareTask(AbstractCompactionTask task) throws InterruptedException {
+ if (task == null) {
+ return false;
+ }
+ if (!checkTaskValid(task)) {
+ dropCompactionTask(task);
+ return false;
+ }
+ if (!task.tryOccupyResourcesForRunning()) {
+ put(task);
+ return false;
+ }
+ if (!transitTaskFileStatus(task)) {
+ dropCompactionTask(task);
+ return false;
+ }
+ return true;
+ }
+
private void dropCompactionTask(AbstractCompactionTask task) {
task.resetCompactionCandidateStatusForAllSourceFiles();
task.handleTaskCleanup();
(iotdb) 03/03: fix(jdbc): when parameter is string ,append "\"" (#12371)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 040ab6505de684a4fd1b36f710e0740dfe4c3715
Author: CritasWang <cr...@outlook.com>
AuthorDate: Fri Apr 19 15:38:44 2024 +0800
fix(jdbc): when parameter is string ,append "\"" (#12371)
---
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 7 ++++++-
.../iotdb/jdbc/IoTDBPreparedStatementTest.java | 23 ++++++++++++++++++++++
2 files changed, 29 insertions(+), 1 deletion(-)
diff --git a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java
index c93c1d55027..6b7bd8e40ac 100644
--- a/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java
+++ b/iotdb-client/jdbc/src/main/java/org/apache/iotdb/jdbc/IoTDBPreparedStatement.java
@@ -880,7 +880,12 @@ public class IoTDBPreparedStatement extends IoTDBStatement implements PreparedSt
@Override
public void setString(int parameterIndex, String x) {
- this.parameters.put(parameterIndex, x);
+ // if the sql is insert and the value is not a string literal, add double quotes
+ if (sql.trim().toUpperCase().startsWith("INSERT") && !x.startsWith("\"") && !x.endsWith("'")) {
+ this.parameters.put(parameterIndex, "\"" + x + "\"");
+ } else {
+ this.parameters.put(parameterIndex, x);
+ }
}
@Override
diff --git a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java
index f236ea12043..b7d8203cf93 100644
--- a/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java
+++ b/iotdb-client/jdbc/src/test/java/org/apache/iotdb/jdbc/IoTDBPreparedStatementTest.java
@@ -375,4 +375,27 @@ public class IoTDBPreparedStatementTest {
"INSERT INTO root.ln.wf01.wt02(time,a,b,c,d,e,f) VALUES(2020-01-01T10:10:10,false,123,123234345,123.423,-1323.0,\"abc\")",
argument.getValue().getStatement());
}
+
+ @Test
+ public void testInsertStatement4() throws Exception {
+ String sql = "INSERT INTO root.ln.wf01.wt02(time,a,b,c,d,e,f) VALUES(?,?,?,?,?,?,?)";
+
+ IoTDBPreparedStatement ps =
+ new IoTDBPreparedStatement(connection, client, sessionId, sql, zoneId);
+ ps.setObject(1, "2020-01-01 10:10:10", Types.TIMESTAMP, -1);
+ ps.setObject(2, false, Types.BOOLEAN, -1);
+ ps.setObject(3, 123, Types.INTEGER, -1);
+ ps.setObject(4, 123234345, Types.BIGINT);
+ ps.setObject(5, 123.423f, Types.FLOAT);
+ ps.setObject(6, -1323.0, Types.DOUBLE);
+ ps.setObject(7, "abc", Types.VARCHAR);
+ ps.execute();
+
+ ArgumentCaptor<TSExecuteStatementReq> argument =
+ ArgumentCaptor.forClass(TSExecuteStatementReq.class);
+ verify(client).executeStatementV2(argument.capture());
+ assertEquals(
+ "INSERT INTO root.ln.wf01.wt02(time,a,b,c,d,e,f) VALUES(2020-01-01T10:10:10,false,123,123234345,123.423,-1323.0,\"abc\")",
+ argument.getValue().getStatement());
+ }
}
(iotdb) 02/03: Do not set read only in compaction module (#12311)
Posted by ha...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch rc/1.3.1
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit a06ff5968946e316647560f6b10953cd0450e39b
Author: shuwenwei <55...@users.noreply.github.com>
AuthorDate: Thu Apr 11 10:52:09 2024 +0800
Do not set read only in compaction module (#12311)
---
.../execute/exception/CompactionExceptionHandler.java | 15 +++++----------
.../execute/recover/CompactionRecoverTask.java | 18 ++++--------------
.../execute/task/AbstractCompactionTask.java | 10 +++++-----
.../execute/task/InnerSpaceCompactionTask.java | 2 +-
.../impl/RewriteCrossSpaceCompactionSelector.java | 2 +-
.../inner/InnerSpaceCompactionExceptionTest.java | 5 +----
6 files changed, 17 insertions(+), 35 deletions(-)
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
index 12039324339..5c663e37955 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/exception/CompactionExceptionHandler.java
@@ -102,23 +102,19 @@ public class CompactionExceptionHandler {
if (!handleSuccess) {
LOGGER.error(
"[Compaction][ExceptionHandler] Fail to handle {} space compaction exception, "
- + "set allowCompaction to false in {}",
+ + "storage group is {}",
compactionType,
fullStorageGroupName);
- tsFileManager.setAllowCompaction(false);
} else {
FileUtils.delete(logFile);
}
} catch (IOException e) {
// catch throwable when handling exception
- // set the allowCompaction to false
LOGGER.error(
"[Compaction][ExceptionHandler] exception occurs when handling exception in {} space compaction. "
- + "Set allowCompaction to false in {}",
+ + "storage group is {}",
compactionType,
- fullStorageGroupName,
- e);
- tsFileManager.setAllowCompaction(false);
+ fullStorageGroupName);
}
}
@@ -209,7 +205,7 @@ public class CompactionExceptionHandler {
/**
* Some source files are lost, check if all target files are complete. If all target files are
* complete, delete the remaining source files and compaction mods files. If some target files are
- * not complete, set the allowCompaction in tsFileManager to false and print some error logs.
+ * not complete, print some error logs.
*
* @throws IOException if the io operations on file fails
*/
@@ -252,8 +248,7 @@ public class CompactionExceptionHandler {
if (!TsFileUtils.isTsFileComplete(targetResource.getTsFile())) {
LOGGER.error(
"{} [Compaction][ExceptionHandler] target file {} is not complete, "
- + "and some source files {} is lost, do nothing. "
- + "Set allowCompaction to false",
+ + "and some source files {} is lost, do nothing.",
fullStorageGroupName,
targetResource,
lostSourceResources);
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java
index 7f238ba02f7..a504df6e7cb 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/recover/CompactionRecoverTask.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.recover;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.storageengine.dataregion.compaction.execute.utils.CompactionUtils;
@@ -113,10 +111,7 @@ public class CompactionRecoverTask {
LOGGER.error("Recover compaction error", e);
} finally {
if (!recoverSuccess) {
- LOGGER.error(
- "{} [Compaction][Recover] Failed to recover compaction, set allowCompaction to false",
- fullStorageGroupName);
- tsFileManager.setAllowCompaction(false);
+ LOGGER.error("{} [Compaction][Recover] Failed to recover compaction", fullStorageGroupName);
} else {
if (compactionLogFile.exists()) {
try {
@@ -127,12 +122,10 @@ public class CompactionRecoverTask {
FileUtils.delete(compactionLogFile);
} catch (IOException e) {
LOGGER.error(
- "{} [Compaction][Recover] Exception occurs while deleting log file {}, "
- + "set allowCompaction to false",
+ "{} [Compaction][Recover] Exception occurs while deleting log file {}",
fullStorageGroupName,
compactionLogFile,
e);
- tsFileManager.setAllowCompaction(false);
}
}
}
@@ -190,8 +183,7 @@ public class CompactionRecoverTask {
CompactionUtils.deleteCompactionModsFile(sourceTsFileResourceList, Collections.emptyList());
} catch (IOException e) {
LOGGER.error(
- "{} [Compaction][Recover] Exception occurs while deleting compaction mods file, "
- + "set allowCompaction to false",
+ "{} [Compaction][Recover] Exception occurs while deleting compaction mods file",
fullStorageGroupName,
e);
return false;
@@ -265,11 +257,9 @@ public class CompactionRecoverTask {
|| !TsFileUtils.isTsFileComplete(new TsFileResource(targetFile).getTsFile())) {
LOGGER.error(
"{} [Compaction][ExceptionHandler] target file {} is not complete, "
- + "and some source files is lost, do nothing. "
- + "Set allowCompaction to false",
+ + "and some source files is lost, do nothing.",
fullStorageGroupName,
targetFileIdentifier.getFilePath());
- CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
return false;
}
}
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
index 7f33d7eaf3f..209680b5a1a 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/AbstractCompactionTask.java
@@ -19,8 +19,6 @@
package org.apache.iotdb.db.storageengine.dataregion.compaction.execute.task;
-import org.apache.iotdb.commons.cluster.NodeStatus;
-import org.apache.iotdb.commons.conf.CommonDescriptor;
import org.apache.iotdb.commons.conf.IoTDBConstant;
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.service.metrics.CompactionMetrics;
@@ -319,9 +317,11 @@ public abstract class AbstractCompactionTask {
dataRegionId,
this,
e);
- tsFileManager.setAllowCompaction(false);
- LOGGER.error("stop compaction because of exception during recovering");
- CommonDescriptor.getInstance().getConfig().setNodeStatus(NodeStatus.ReadOnly);
+ // Do not set allow compaction to false here. To keep the error environment, mark all source
+ // files in memory to avoid compaction.
+ for (TsFileResource sourceTsFileResource : getAllSourceTsFiles()) {
+ sourceTsFileResource.setTsFileRepairStatus(TsFileRepairStatus.CAN_NOT_REPAIR);
+ }
}
protected void insertFilesToTsFileManager(List<TsFileResource> tsFiles) throws IOException {
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
index 7fcc419db46..9de3917a9ae 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InnerSpaceCompactionTask.java
@@ -172,7 +172,7 @@ public class InnerSpaceCompactionTask extends AbstractCompactionTask {
recoverMemoryStatus = true;
LOGGER.info(
"{}-{} [Compaction] {} InnerSpaceCompaction task starts with {} files, "
- + "total file size is {} MB, memory cost is {} MB",
+ + "total file size is {} MB, estimated memory cost is {} MB",
storageGroupName,
dataRegionId,
sequence ? "Sequence" : "Unsequence",
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
index fdc1ea8fbf4..64807c0c913 100644
--- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
+++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/selector/impl/RewriteCrossSpaceCompactionSelector.java
@@ -362,7 +362,7 @@ public class RewriteCrossSpaceCompactionSelector implements ICrossSpaceSelector
"{} [{}] Total source files: {} seqFiles, {} unseqFiles. "
+ "Candidate source files: {} seqFiles, {} unseqFiles. "
+ "Selected source files: {} seqFiles, "
- + "{} unseqFiles, total memory cost {} MB, "
+ + "{} unseqFiles, estimated memory cost {} MB, "
+ "total selected file size is {} MB, "
+ "total selected seq file size is {} MB, "
+ "total selected unseq file size is {} MB, "
diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java
index aa54be5766a..1087c56863b 100644
--- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java
+++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionExceptionTest.java
@@ -215,8 +215,7 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact
}
/**
- * Test some source files are lost and target file is not complete. System should be set to read
- * only at this time.
+ * Test some source files are lost and target file is not complete.
*
* @throws Exception
*/
@@ -265,8 +264,6 @@ public class InnerSpaceCompactionExceptionTest extends AbstractInnerSpaceCompact
Assert.assertTrue(resource.resourceFileExists());
Assert.assertTrue(resource.getTsFile().exists());
}
-
- Assert.assertFalse(tsFileManager.isAllowCompaction());
}
/**