You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by xu...@apache.org on 2022/01/19 04:49:35 UTC
[iotdb] 03/03: merge and fix typo
This is an automated email from the ASF dual-hosted git repository.
xuekaifeng pushed a commit to branch xkf_idtable_recovery
in repository https://gitbox.apache.org/repos/asf/iotdb.git
commit 1cae8a31010bb4379a17d02a78b2c076f9b4178d
Merge: b49bb01 24d4b97
Author: 151250176 <15...@smail.nju.edu.cn>
AuthorDate: Wed Jan 19 12:03:25 2022 +0800
merge and fix typo
.../org/apache/iotdb/db/qp/sql/IoTDBSqlParser.g4 | 6 +-
docs/UserGuide/Maintenance-Tools/Metric-Tool.md | 53 +-
docs/zh/SystemDesign/StorageEngine/MergeManager.md | 7 +-
docs/zh/UserGuide/Maintenance-Tools/Metric-Tool.md | 53 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 9 +-
.../db/integration/IoTDBNewTsFileCompactionIT.java | 1 +
.../integration/IoTDBSizeTieredCompactionIT.java | 7 +-
.../aligned/IOTDBInsertAlignedValuesIT.java | 3 +
.../iotdb/metrics/dropwizard/MetricName.java | 12 +-
.../prometheus/DropwizardMetricsExporter.java | 3 +-
.../main/assembly/resources/conf/iotdb-metric.yml | 6 +-
.../metrics/config/MetricConfigDescriptor.java | 2 +-
.../interface/src/test/resources/iotdb-metric.yml | 4 +-
pom.xml | 2 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 34 +-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 7 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 9 -
.../org/apache/iotdb/db/engine/StorageEngine.java | 4 +-
.../db/engine/cache/TimeSeriesMetadataCache.java | 18 +-
.../db/engine/compaction/CompactionScheduler.java | 23 +-
.../engine/compaction/CompactionTaskManager.java | 30 +
.../db/engine/compaction/CompactionUtils.java | 388 +++
.../AbstractCrossSpaceCompactionSelector.java | 10 +-
.../cross/AbstractCrossSpaceCompactionTask.java | 1 -
.../compaction/cross/CrossCompactionStrategy.java | 45 +-
.../CrossSpaceCompactionExceptionHandler.java | 256 ++
.../cross/CrossSpaceCompactionTaskFactory.java | 16 +-
.../inplace/InplaceCompactionRecoverTask.java | 110 -
.../cross/inplace/InplaceCompactionTask.java | 244 --
.../inplace/manage/CrossSpaceMergeContext.java | 88 -
.../cross/inplace/manage/MergeFuture.java | 151 -
.../cross/inplace/manage/MergeManager.java | 365 ---
.../cross/inplace/manage/MergeManagerMBean.java | 26 -
.../cross/inplace/manage/MergeThreadPool.java | 54 -
.../recover/InplaceCompactionLogAnalyzer.java | 353 ---
.../inplace/recover/InplaceCompactionLogger.java | 150 -
.../task/CleanLastCrossSpaceCompactionTask.java | 67 -
.../cross/inplace/task/CrossSpaceMergeTask.java | 259 --
.../cross/inplace/task/MergeCallback.java | 40 -
.../cross/inplace/task/MergeFileTask.java | 405 ---
.../cross/inplace/task/MergeMultiChunkTask.java | 709 -----
.../RewriteCrossSpaceCompactionSelector.java} | 25 +-
.../manage/CrossSpaceMergeResource.java | 103 +-
.../RewriteCrossSpaceCompactionLogAnalyzer.java | 118 +
.../recover/RewriteCrossSpaceCompactionLogger.java | 86 +
.../selector/ICrossSpaceMergeFileSelector.java | 2 +-
.../selector/IFileQueryMemMeasurement.java | 2 +-
.../selector/IMergePathSelector.java | 2 +-
.../selector/MaxFileMergeFileSelector.java | 4 +-
.../selector/MaxSeriesMergeFileSelector.java | 4 +-
.../selector/MergeFileStrategy.java | 2 +-
.../selector/NaivePathSelector.java | 2 +-
.../task/RewriteCrossCompactionRecoverTask.java} | 180 +-
.../task/RewriteCrossSpaceCompactionTask.java | 296 ++
.../AbstractInnerSpaceCompactionSelector.java | 7 +-
.../compaction/inner/InnerCompactionStrategy.java | 11 +-
.../InnerSpaceCompactionExceptionHandler.java | 14 +-
.../inner/InnerSpaceCompactionTaskFactory.java | 1 -
.../SizeTieredCompactionRecoverTask.java | 16 +-
.../sizetiered/SizeTieredCompactionSelector.java | 3 -
.../inner/sizetiered/SizeTieredCompactionTask.java | 31 +-
.../utils/AlignedSeriesCompactionExecutor.java | 3 +
.../inner/utils/InnerSpaceCompactionUtils.java | 22 +-
.../inner/utils/MultiTsFileDeviceIterator.java | 58 +-
.../utils/SingleSeriesCompactionExecutor.java | 11 +-
.../compaction/task/AbstractCompactionTask.java | 4 +-
.../compaction/task/CompactionRecoverTask.java | 30 +-
.../writer/AbstractCompactionWriter.java | 157 +
.../writer/CrossSpaceCompactionWriter.java | 182 ++
.../writer/InnerSpaceCompactionWriter.java | 86 +
.../engine/storagegroup/TsFileNameGenerator.java | 84 +-
.../db/engine/storagegroup/TsFileResource.java | 5 +
.../storagegroup/VirtualStorageGroupProcessor.java | 119 +-
.../virtualSg/StorageGroupManager.java | 4 +-
.../org/apache/iotdb/db/metadata/MManager.java | 44 +-
.../db/metadata/idtable/entry/DeviceEntry.java | 3 +-
.../db/metadata/idtable/entry/DeviceIDFactory.java | 4 +-
.../db/metadata/idtable/entry/SHA256DeviceID.java | 6 +-
.../apache/iotdb/db/qp/executor/PlanExecutor.java | 60 +-
.../db/qp/logical/sys/ShowMergeStatusOperator.java | 40 -
.../db/qp/physical/sys/ShowMergeStatusPlan.java | 27 -
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 1 -
.../apache/iotdb/db/qp/sql/IoTDBSqlVisitor.java | 8 -
.../db/query/control/QueryResourceManager.java | 17 +
.../java/org/apache/iotdb/db/service/IoTDB.java | 2 -
.../db/sync/sender/manage/SyncFileManager.java | 4 +-
.../java/org/apache/iotdb/db/utils/MergeUtils.java | 127 +-
.../engine/compaction/AbstractCompactionTest.java | 292 ++
.../engine/compaction/CompactionSchedulerTest.java | 70 +-
.../compaction/CompactionTaskManagerTest.java | 63 +-
.../db/engine/compaction/CompactionUtilsTest.java | 3344 ++++++++++++++++++++
.../cross/CrossSpaceCompactionExceptionTest.java | 498 +++
.../compaction/cross/CrossSpaceCompactionTest.java | 411 ++-
.../cross/MaxFileMergeFileSelectorTest.java | 6 +-
.../cross/MaxSeriesMergeFileSelectorTest.java | 4 +-
.../db/engine/compaction/cross/MergeLogTest.java | 92 -
.../engine/compaction/cross/MergeManagerTest.java | 146 -
.../engine/compaction/cross/MergeOverLapTest.java | 205 --
.../db/engine/compaction/cross/MergePerfTest.java | 108 -
.../db/engine/compaction/cross/MergeTaskTest.java | 575 ----
.../db/engine/compaction/cross/MergeTest.java | 3 -
.../engine/compaction/cross/MergeUpgradeTest.java | 4 +-
.../RewriteCrossSpaceCompactionRecoverTest.java | 506 +++
.../cross/RewriteCrossSpaceCompactionTest.java | 334 ++
.../inner/AbstractInnerSpaceCompactionTest.java | 26 +-
.../inner/InnerCompactionSchedulerTest.java | 87 +-
.../compaction/inner/InnerCompactionTest.java | 2 +-
.../compaction/inner/InnerSeqCompactionTest.java | 2 +-
.../inner/InnerSpaceCompactionExceptionTest.java | 46 +-
.../inner/InnerSpaceCompactionUtilsOldTest.java | 2 +-
.../compaction/inner/InnerUnseqCompactionTest.java | 45 +-
.../SizeTieredCompactionHandleExceptionTest.java | 43 +-
.../SizeTieredCompactionRecoverTest.java | 58 +-
.../recover/SizeTieredCompactionRecoverTest.java | 128 +-
.../task/FakedCrossSpaceCompactionTask.java | 21 +-
.../task/FakedCrossSpaceCompactionTaskFactory.java | 16 +-
.../task/FakedInnerSpaceCompactionTask.java | 3 -
.../task/FakedInnerSpaceCompactionTaskFactory.java | 1 -
.../compaction/utils/CompactionClearUtils.java | 72 +-
.../utils/CompactionFileGeneratorUtils.java | 65 +-
.../engine/storagegroup/FakedTsFileResource.java | 1 +
.../storagegroup/StorageGroupProcessorTest.java | 2 +-
.../db/metadata/idtable/IDTableRecoverTest.java | 12 +-
.../query/reader/series/SeriesReaderTestUtil.java | 2 -
.../iotdb/tsfile/file/header/ChunkHeader.java | 4 +-
.../file/metadata/AlignedTimeSeriesMetadata.java | 6 +-
.../read/TsFileAlignedSeriesReaderIterator.java | 3 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../iotdb/tsfile/utils/TsFileGeneratorUtils.java | 285 ++
.../tsfile/write/chunk/AlignedChunkWriterImpl.java | 17 +
.../iotdb/tsfile/write/chunk/ValueChunkWriter.java | 17 +
.../apache/iotdb/tsfile/read/TsFileReaderTest.java | 6 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 83 +-
.../iotdb/tsfile/write/TsFileWriteApiTest.java | 46 +-
134 files changed, 8307 insertions(+), 5504 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java
index 7f26357,de52b8a..e5cbd90
--- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceEntry.java
@@@ -19,10 -19,8 +19,11 @@@
package org.apache.iotdb.db.metadata.idtable.entry;
++import org.apache.iotdb.db.utils.TestOnly;
++
import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
- import org.apache.iotdb.db.utils.TestOnly;
/** device entry in id table */
public class DeviceEntry {
diff --cc server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java
index c4a9676,c4a9676..6ebee8f
--- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/DeviceIDFactory.java
@@@ -89,9 -89,9 +89,9 @@@ public class DeviceIDFactory
.getConfig()
.getDeviceIDTransformationMethod()
.equals("SHA256")) {
-- getDeviceIDFunction = partialPath -> new SHA256DeviceID(partialPath.toString());
++ getDeviceIDFunction = SHA256DeviceID::new;
} else {
-- getDeviceIDFunction = partialPath -> new PlainDeviceID(partialPath.toString());
++ getDeviceIDFunction = PlainDeviceID::new;
}
}
}
diff --cc server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java
index 7cb477e,7cb477e..62479bb
--- a/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/idtable/entry/SHA256DeviceID.java
@@@ -34,7 -34,7 +34,7 @@@ public class SHA256DeviceID implements
long l3;
long l4;
-- private static final String SEPERATOR = "#";
++ private static final String SEPARATOR = "#";
/** using lots of message digest for improving parallelism */
private static MessageDigest[] md;
@@@ -71,7 -71,7 +71,7 @@@
* @param deviceID a sha 256 string
*/
private void fromSHA256String(String deviceID) {
-- String[] part = deviceID.split(SEPERATOR);
++ String[] part = deviceID.split(SEPARATOR);
l1 = Long.parseLong(part[0]);
l2 = Long.parseLong(part[1]);
l3 = Long.parseLong(part[2]);
@@@ -143,6 -143,6 +143,6 @@@
@Override
public String toStringID() {
-- return l1 + SEPERATOR + l2 + SEPERATOR + l3 + SEPERATOR + l4;
++ return l1 + SEPARATOR + l2 + SEPARATOR + l3 + SEPARATOR + l4;
}
}
diff --cc server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java
index 379caeb,0000000..e73eafe
mode 100644,000000..100644
--- a/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java
+++ b/server/src/test/java/org/apache/iotdb/db/metadata/idtable/IDTableRecoverTest.java
@@@ -1,209 -1,0 +1,211 @@@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iotdb.db.metadata.idtable;
+
- import static org.junit.Assert.assertEquals;
-
- import java.io.IOException;
- import java.util.ArrayList;
- import java.util.List;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.StorageEngineException;
+import org.apache.iotdb.db.exception.metadata.IllegalPathException;
+import org.apache.iotdb.db.exception.query.QueryProcessException;
+import org.apache.iotdb.db.metadata.idtable.entry.DeviceEntry;
+import org.apache.iotdb.db.metadata.path.PartialPath;
+import org.apache.iotdb.db.qp.Planner;
+import org.apache.iotdb.db.qp.executor.PlanExecutor;
+import org.apache.iotdb.db.qp.physical.PhysicalPlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertRowPlan;
+import org.apache.iotdb.db.qp.physical.crud.InsertTabletPlan;
+import org.apache.iotdb.db.utils.EnvironmentUtils;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.utils.Binary;
++
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
++import java.io.IOException;
++import java.util.ArrayList;
++import java.util.List;
++
++import static org.junit.Assert.assertEquals;
++
+public class IDTableRecoverTest {
+ private final Planner processor = new Planner();
+
+ private boolean isEnableIDTable = false;
+
+ private String originalDeviceIDTransformationMethod = null;
+
+ private boolean isEnableIDTableLogFile = false;
+
+ @Before
+ public void before() {
+ IoTDBDescriptor.getInstance().getConfig().setAutoCreateSchemaEnabled(true);
+ isEnableIDTable = IoTDBDescriptor.getInstance().getConfig().isEnableIDTable();
+ originalDeviceIDTransformationMethod =
+ IoTDBDescriptor.getInstance().getConfig().getDeviceIDTransformationMethod();
+ isEnableIDTableLogFile = IoTDBDescriptor.getInstance().getConfig().isEnableIDTableLogFile();
+
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(true);
+ IoTDBDescriptor.getInstance().getConfig().setDeviceIDTransformationMethod("SHA256");
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(true);
+ EnvironmentUtils.envSetUp();
+ }
+
+ @After
+ public void clean() throws IOException, StorageEngineException {
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTable(isEnableIDTable);
+ IoTDBDescriptor.getInstance()
+ .getConfig()
+ .setDeviceIDTransformationMethod(originalDeviceIDTransformationMethod);
+ IoTDBDescriptor.getInstance().getConfig().setEnableIDTableLogFile(isEnableIDTableLogFile);
+
+ EnvironmentUtils.cleanEnv();
+ }
+
+ @Test
+ public void testRecover() throws Exception {
+ insertDataInMemoryWithTablet(false);
+ insertDataInMemoryWithRecord(false);
+
+ PlanExecutor executor = new PlanExecutor();
+ PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush");
+ executor.processNonQuery(flushPlan);
+
+ IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp"));
+ List<DeviceEntry> memoryList = idTable.getAllDeviceEntry();
+
+ // restart
+ try {
+ EnvironmentUtils.restartDaemon();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+
+ // check id table fields
+
+ idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1"));
+ List<DeviceEntry> recoverList = idTable.getAllDeviceEntry();
+
+ assertEquals(memoryList, recoverList);
+ }
+
+ @Test
+ public void testRecoverAligned() throws Exception {
+ insertDataInMemoryWithTablet(true);
+ insertDataInMemoryWithRecord(false);
+
+ PlanExecutor executor = new PlanExecutor();
+ PhysicalPlan flushPlan = processor.parseSQLToPhysicalPlan("flush");
+ executor.processNonQuery(flushPlan);
+
+ IDTable idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp"));
+ List<DeviceEntry> memoryList = idTable.getAllDeviceEntry();
+
+ // restart
+ try {
+ EnvironmentUtils.restartDaemon();
+ } catch (Exception e) {
+ Assert.fail();
+ }
+
+ // check id table fields
+
+ idTable = IDTableManager.getInstance().getIDTable(new PartialPath("root.isp.d1"));
+ List<DeviceEntry> recoverList = idTable.getAllDeviceEntry();
+
+ assertEquals(memoryList, recoverList);
+ }
+
+ private void insertDataInMemoryWithRecord(boolean isAligned)
+ throws IllegalPathException, QueryProcessException {
+ long time = 100L;
+ TSDataType[] dataTypes =
+ new TSDataType[] {
+ TSDataType.DOUBLE,
+ TSDataType.FLOAT,
+ TSDataType.INT64,
+ TSDataType.INT32,
+ TSDataType.BOOLEAN,
+ TSDataType.TEXT
+ };
+
+ String[] columns = new String[6];
+ columns[0] = 1.0 + "";
+ columns[1] = 2 + "";
+ columns[2] = 10000 + "";
+ columns[3] = 100 + "";
+ columns[4] = false + "";
+ columns[5] = "hh" + 0;
+
+ InsertRowPlan insertRowPlan =
+ new InsertRowPlan(
+ new PartialPath("root.isp.d1"),
+ time,
+ new String[] {"s1", "s2", "s3", "s4", "s5", "s6"},
+ dataTypes,
+ columns);
+ insertRowPlan.setAligned(isAligned);
+
+ PlanExecutor executor = new PlanExecutor();
+ executor.insert(insertRowPlan);
+ }
+
+ private void insertDataInMemoryWithTablet(boolean isAligned)
+ throws IllegalPathException, QueryProcessException {
+ long[] times = new long[] {110L, 111L, 112L, 113L};
+ List<Integer> dataTypes = new ArrayList<>();
+ dataTypes.add(TSDataType.DOUBLE.ordinal());
+ dataTypes.add(TSDataType.FLOAT.ordinal());
+ dataTypes.add(TSDataType.INT64.ordinal());
+ dataTypes.add(TSDataType.INT32.ordinal());
+ dataTypes.add(TSDataType.BOOLEAN.ordinal());
+ dataTypes.add(TSDataType.TEXT.ordinal());
+
+ Object[] columns = new Object[6];
+ columns[0] = new double[4];
+ columns[1] = new float[4];
+ columns[2] = new long[4];
+ columns[3] = new int[4];
+ columns[4] = new boolean[4];
+ columns[5] = new Binary[4];
+
+ for (int r = 0; r < 4; r++) {
+ ((double[]) columns[0])[r] = 10.0 + r;
+ ((float[]) columns[1])[r] = 20 + r;
+ ((long[]) columns[2])[r] = 100000 + r;
+ ((int[]) columns[3])[r] = 1000 + r;
+ ((boolean[]) columns[4])[r] = false;
+ ((Binary[]) columns[5])[r] = new Binary("mm" + r);
+ }
+
+ InsertTabletPlan tabletPlan =
+ new InsertTabletPlan(
+ new PartialPath("root.isp.d2"),
+ new String[] {"s1", "s2", "s3", "s4", "s5", "s6"},
+ dataTypes);
+ tabletPlan.setTimes(times);
+ tabletPlan.setColumns(columns);
+ tabletPlan.setRowCount(times.length);
+ tabletPlan.setAligned(isAligned);
+
+ PlanExecutor executor = new PlanExecutor();
+ executor.insertTablet(tabletPlan);
+ }
+}