You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by yu...@apache.org on 2021/03/24 03:05:22 UTC
[iotdb] branch rel/0.11 updated: revert
This is an automated email from the ASF dual-hosted git repository.
yuyuankang pushed a commit to branch rel/0.11
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/rel/0.11 by this push:
new 62fecbe revert
62fecbe is described below
commit 62fecbed50fbd0f3f5a592151fee13a204e27344
Author: Ring-k <yu...@hotmail.com>
AuthorDate: Wed Mar 24 11:04:52 2021 +0800
revert
---
.../db/engine/storagegroup/TsFileResource.java | 16 +-
.../iotdb/db/query/executor/LastQueryExecutor.java | 19 +-
.../IoTDBLoadExternalTsfileWithTimePartition.java | 201 ---------------------
3 files changed, 15 insertions(+), 221 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
index 2433259..2cca8d7 100644
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/TsFileResource.java
@@ -717,8 +717,8 @@ public class TsFileResource {
*/
public long getTimePartitionWithCheck() throws PartitionViolationException {
long partitionId = -1;
- for (int index : deviceToIndex.values()) {
- long p = StorageEngine.getTimePartition(startTimes[index]);
+ for (Long startTime : startTimes) {
+ long p = StorageEngine.getTimePartition(startTime);
if (partitionId == -1) {
partitionId = p;
} else {
@@ -726,9 +726,15 @@ public class TsFileResource {
throw new PartitionViolationException(this);
}
}
- p = StorageEngine.getTimePartition(endTimes[index]);
- if (partitionId != p) {
- throw new PartitionViolationException(this);
+ }
+ for (Long endTime : endTimes) {
+ long p = StorageEngine.getTimePartition(endTime);
+ if (partitionId == -1) {
+ partitionId = p;
+ } else {
+ if (partitionId != p) {
+ throw new PartitionViolationException(this);
+ }
}
}
if (partitionId == -1) {
diff --git a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
index 8821cf5..7360a18 100644
--- a/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
+++ b/server/src/main/java/org/apache/iotdb/db/query/executor/LastQueryExecutor.java
@@ -133,7 +133,7 @@ public class LastQueryExecutor {
List<TSDataType> restDataTypes = new ArrayList<>();
List<PartialPath> restPaths = new ArrayList<>();
List<Pair<Boolean, TimeValuePair>> resultContainer =
- readLastPairsFromCache(seriesPaths, dataTypes, filter, cacheAccessors, restPaths, restDataTypes, lastQueryPlan.isDebug());
+ readLastPairsFromCache(seriesPaths, dataTypes, filter, cacheAccessors, restPaths, restDataTypes);
// If any '>' or '>=' filters are specified, only access cache to get Last result.
if (filter != null || restPaths.isEmpty()) {
return resultContainer;
@@ -162,13 +162,7 @@ public class LastQueryExecutor {
resultContainer.get(i).left = true;
if (lastCacheEnabled) {
cacheAccessors.get(i).write(resultContainer.get(i).right);
- if (lastQueryPlan.isDebug()) {
- DEBUG_LOGGER.info(
- "[LastQueryExecutor] Update last cache for path: "
- + seriesPaths
- + " with timestamp: "
- + resultContainer.get(i).right.getTimestamp());
- }
+ DEBUG_LOGGER.info("[LastQueryExecutor] Update last cache for path: " + seriesPaths + " with timestamp: " + resultContainer.get(i).right.getTimestamp());
}
}
}
@@ -179,7 +173,7 @@ public class LastQueryExecutor {
private static List<Pair<Boolean, TimeValuePair>> readLastPairsFromCache(
List<PartialPath> seriesPaths, List<TSDataType> dataTypes, Filter filter,
List<LastCacheAccessor> cacheAccessors,
- List<PartialPath> restPaths, List<TSDataType> restDataTypes, boolean isDebug) {
+ List<PartialPath> restPaths, List<TSDataType> restDataTypes) {
List<Pair<Boolean, TimeValuePair>> resultContainer = new ArrayList<>();
if (lastCacheEnabled) {
for (PartialPath path : seriesPaths) {
@@ -193,12 +187,7 @@ public class LastQueryExecutor {
TimeValuePair tvPair = cacheAccessors.get(i).read();
if (tvPair != null) {
resultContainer.add(new Pair<>(true, tvPair));
- if (isDebug) {
- DEBUG_LOGGER.info(
- "[LastQueryExecutor] Last cache hit for path: {} with timestamp: {}",
- seriesPaths.get(i),
- tvPair.getTimestamp());
- }
+ DEBUG_LOGGER.info("[LastQueryExecutor] Last cache hit for path: {} with timestamp: {}" , seriesPaths.get(i), tvPair.getTimestamp());
} else {
resultContainer.add(new Pair<>(false, null));
restPaths.add(seriesPaths.get(i));
diff --git a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartition.java b/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartition.java
deleted file mode 100644
index ecdbc97..0000000
--- a/server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileWithTimePartition.java
+++ /dev/null
@@ -1,201 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.iotdb.db.integration;
-
-import java.io.File;
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Arrays;
-import org.apache.iotdb.db.conf.IoTDBConfig;
-import org.apache.iotdb.db.conf.IoTDBDescriptor;
-import org.apache.iotdb.db.engine.compaction.CompactionStrategy;
-import org.apache.iotdb.db.exception.metadata.IllegalPathException;
-import org.apache.iotdb.db.metadata.PartialPath;
-import org.apache.iotdb.db.utils.EnvironmentUtils;
-import org.apache.iotdb.db.utils.FileUtils;
-import org.apache.iotdb.jdbc.Config;
-import org.apache.iotdb.tsfile.exception.write.WriteProcessException;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
-import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
-import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
-import org.apache.iotdb.tsfile.read.common.Path;
-import org.apache.iotdb.tsfile.write.TsFileWriter;
-import org.apache.iotdb.tsfile.write.record.TSRecord;
-import org.apache.iotdb.tsfile.write.record.datapoint.DataPoint;
-import org.apache.iotdb.tsfile.write.record.datapoint.LongDataPoint;
-import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
-import org.apache.iotdb.tsfile.write.writer.TsFileIOWriter;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.Test;
-
-public class IoTDBLoadExternalTsfileWithTimePartition {
-
- String DOT = ".";
- String tempDir = "temp";
-
- String STORAGE_GROUP = "root.ln";
- String[] devices = new String[]{"d1", "d2"};
- String[] measurements = new String[]{"s1", "s2"};
-
- // generate several tsfiles, with timestamp from startTime(inclusive) to endTime(exclusive)
- long startTime = 0;
- long endTime = 1000_000;
-
- long timePartition = 100;// unit s
-
- boolean originalIsEnablePartition;
- long originalPartitionInterval;
-
- IoTDBConfig config = IoTDBDescriptor.getInstance().getConfig();
-
- @Before
- public void setUp() throws Exception {
- originalIsEnablePartition = config.isEnablePartition();
- originalPartitionInterval = config.getPartitionInterval();
- config.setEnablePartition(true);
- config.setPartitionInterval(timePartition);
- EnvironmentUtils.closeStatMonitor();
- EnvironmentUtils.envSetUp();
- Class.forName(Config.JDBC_DRIVER_NAME);
- prepareData();
- }
-
- @After
- public void tearDown() throws Exception {
- EnvironmentUtils.cleanEnv();
- IoTDBDescriptor.getInstance()
- .getConfig()
- .setCompactionStrategy(CompactionStrategy.LEVEL_COMPACTION);
- config.setEnablePartition(originalIsEnablePartition);
- config.setPartitionInterval(originalPartitionInterval);
- File f = new File(tempDir);
- if (f.exists()) {
- FileUtils.deleteDirectory(f);
- }
- }
-
- /**
- * get the name of tsfile given counter
- */
- String getName(int counter) {
- return tempDir + File.separator + System.currentTimeMillis() + "-" + counter
- + "-0.tsfile";
- }
-
- /**
- * write a record, given timestamp
- */
- void writeData(TsFileWriter tsFileWriter, long timestamp)
- throws IOException, WriteProcessException {
- for (String deviceId : devices) {
- TSRecord tsRecord = new TSRecord(timestamp,
- STORAGE_GROUP + DOT + deviceId);
- for (String measurement : measurements) {
- DataPoint dPoint = new LongDataPoint(measurement, 10000);
- tsRecord.addTuple(dPoint);
- }
- tsFileWriter.write(tsRecord);
- }
- }
-
- /**
- * register all timeseries in tsfiles
- */
- void register(TsFileWriter tsFileWriter) {
- try {
- for (String deviceId : devices) {
- for (String measurement : measurements) {
- tsFileWriter.registerTimeseries(
- new Path(STORAGE_GROUP + DOT + deviceId, measurement),
- new MeasurementSchema(measurement, TSDataType.INT64, TSEncoding.RLE));
- }
- }
- } catch (WriteProcessException e) {
- e.printStackTrace();
- }
- }
-
- /**
- * create multiple tsfiles, each correspond to a time partition.
- */
- private void prepareData() throws IOException {
- File dir = new File(tempDir);
- if (dir.exists()) {
- FileUtils.deleteDirectory(dir);
- }
- dir.mkdir();
- try {
- File f;
- TsFileWriter tsFileWriter = null;
- int counter = 0;
- for (long timestamp = startTime; timestamp < endTime; timestamp += 1000) {
- if (timestamp % (timePartition*1000) == 0) {
- if (tsFileWriter != null) {
- tsFileWriter.flushAllChunkGroups();
- tsFileWriter.close();
- counter++;
- }
- String path = getName(counter);
- f = FSFactoryProducer.getFSFactory().getFile(path);
- tsFileWriter = new TsFileWriter(new TsFileIOWriter(f));
- register(tsFileWriter);
- }
- writeData(tsFileWriter, timestamp);
- }
- tsFileWriter.flushAllChunkGroups();
- tsFileWriter.close();
- } catch (Throwable e) {
- e.printStackTrace();
- }
-
- }
-
-
- @Test
- public void loadTsfileWithTimePartition() {
- try (Connection connection =
- DriverManager.getConnection("jdbc:iotdb://127.0.0.1:6667/", "root", "root");
- Statement statement = connection.createStatement()) {
-
- statement.execute(String.format("load \"%s\"", new File(tempDir).getAbsolutePath()));
-
- String dataDir = config.getDataDirs()[0];
- File f = new File(dataDir,
- new PartialPath("sequence") + File.separator + "root.ln");
- System.out.println(Arrays.toString(f.list()));
- Assert.assertEquals((endTime - startTime) / (timePartition * 1000), f.list().length);
-
- for (int i = 0; i < (endTime - startTime) / (timePartition * 1000); i++) {
- Assert.assertEquals("" + i, f.list()[i]);
- }
- // each time partition folder should contain 2 files, the tsfile and the resource file
- for (int i = 0; i < (endTime - startTime) / (timePartition * 1000); i++) {
- Assert.assertEquals(2, new File(f.getAbsolutePath(), "" + i).list().length);
- }
- } catch (SQLException | IllegalPathException throwables) {
- throwables.printStackTrace();
- }
- }
-
-}
\ No newline at end of file