You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ji...@apache.org on 2019/09/17 12:37:20 UTC
[incubator-iotdb] 01/01: Merge branch 'master' into dev_TTL
This is an automated email from the ASF dual-hosted git repository.
jiangtian pushed a commit to branch dev_TTL
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit b381c8ef47db9a6fe21b283341373f195f01a97f
Merge: 75e4eb5 18da903
Author: jt <jt...@163.com>
AuthorDate: Tue Sep 17 20:26:17 2019 +0800
Merge branch 'master' into dev_TTL
# Conflicts:
# server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
# server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
# server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
README.md | 222 ++++++++---
.../org/apache/iotdb/client/AbstractClient.java | 13 -
.../org/apache/iotdb/client/AbstractClientIT.java | 5 -
docs/Documentation-CHN/QuickStart.md | 259 +++++++------
.../4-Deployment and Management/2-Configuration.md | 27 ++
.../UserGuide/7-Session API/1-Session API.md | 7 +
.../{9-Tools-spark.md => 9-Tools-spark-iotdb.md} | 0
.../{9-Tools-spark.md => 9-Tools-spark-tsfile.md} | 0
docs/Documentation/QuickStart.md | 189 +++++++--
.../4-Deployment and Management/2-Configuration.md | 27 ++
.../UserGuide/7-Session API/1-Session API.md | 7 +
docs/Documentation/UserGuide/8-TsFile/2-Usage.md | 13 +
.../Documentation/UserGuide/9-Tools-spark-iotdb.md | 171 +++++++++
.../{9-Tools-spark.md => 9-Tools-spark-tsfile.md} | 0
.../main/java/org/apache/iotdb/SessionExample.java | 11 +-
.../apache/iotdb/tsfile/TsFileSequenceRead.java | 3 +-
.../iotdb/tsfile/TsFileWriteWithRowBatch.java | 4 +-
.../iotdb/tsfile/TsFileWriteWithTSRecord.java | 5 +-
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 20 +-
.../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 59 ++-
pom.xml | 1 +
server/pom.xml | 19 -
.../resources/conf/iotdb-engine.properties | 10 +-
.../src/assembly/resources/sbin/start-server.bat | 6 +-
.../iotdb/db/auth/role/LocalFileRoleAccessor.java | 19 +-
.../iotdb/db/auth/user/LocalFileUserAccessor.java | 38 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 90 ++++-
.../org/apache/iotdb/db/conf/IoTDBConfigCheck.java | 8 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 11 +-
.../iotdb/db/conf/adapter/CompressionRatio.java | 7 +-
.../db/conf/directories/DirectoryManager.java | 13 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 4 +-
.../iotdb/db/engine/fileSystem/HdfsFile.java | 156 --------
.../{FileFactory.java => SystemFileFactory.java} | 26 +-
.../db/engine/merge/manage/MergeResource.java | 4 +-
.../iotdb/db/engine/merge/recover/LogAnalyzer.java | 8 +-
.../db/engine/modification/ModificationFile.java | 5 +-
.../io/LocalTextModificationAccessor.java | 25 +-
.../engine/storagegroup/StorageGroupProcessor.java | 27 +-
.../db/engine/storagegroup/TsFileResource.java | 12 +-
.../version/SimpleFileVersionController.java | 9 +-
.../org/apache/iotdb/db/metadata/MManager.java | 9 +-
.../iotdb/db/monitor/collector/FileSize.java | 5 +-
.../iotdb/db/qp/physical/crud/DeletePlan.java | 4 +-
.../impl/{MeanAggrFunc.java => AvgAggrFunc.java} | 8 +-
.../db/query/aggregation/impl/SumAggrFunc.java | 2 +-
.../iotdb/db/query/factory/AggreFuncFactory.java | 6 +-
.../java/org/apache/iotdb/db/service/Monitor.java | 5 +-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 25 +-
.../iotdb/db/sync/receiver/SyncServiceImpl.java | 21 +-
.../apache/iotdb/db/tools/MemEst/MemEstTool.java | 5 +-
.../iotdb/db/tools/TsFileResourcePrinter.java | 6 +-
.../java/org/apache/iotdb/db/tools/WalChecker.java | 6 +-
.../org/apache/iotdb/db/utils/CommonUtils.java | 10 +-
.../org/apache/iotdb/db/writelog/io/LogWriter.java | 3 +-
.../db/writelog/node/ExclusiveWriteLogNode.java | 12 +-
.../iotdb/db/writelog/recover/LogReplayer.java | 5 +-
.../writelog/recover/TsFileRecoverPerformer.java | 7 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 4 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 8 +-
.../db/engine/modification/DeletionQueryTest.java | 1 -
.../engine/storagegroup/TsFileProcessorTest.java | 12 +-
.../org/apache/iotdb/db/integration/Constant.java | 4 +-
.../iotdb/db/integration/IOTDBGroupByIT.java | 12 +-
.../iotdb/db/integration/IoTDBAggregationIT.java | 23 +-
.../integration/IoTDBAggregationLargeDataIT.java | 28 +-
.../integration/IoTDBAggregationSmallDataIT.java | 12 +-
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 16 +-
.../db/query/control/FileReaderManagerTest.java | 8 +-
.../iotdb/db/writelog/recover/LogReplayerTest.java | 6 +-
.../db/writelog/recover/SeqTsFileRecoverTest.java | 4 +-
.../writelog/recover/UnseqTsFileRecoverTest.java | 4 +-
service-rpc/rpc-changelist.md | 6 +-
service-rpc/src/main/thrift/rpc.thrift | 11 +-
session/pom.xml | 12 +
.../main/java/org/apache/iotdb/session/Config.java | 2 +-
.../java/org/apache/iotdb/session/Session.java | 44 ++-
.../org/apache/iotdb/session/IoTDBSessionIT.java | 98 ++++-
.../iotdb/session/utils/EnvironmentUtils.java | 192 ++++++++++
spark-iotdb-connector/Readme.md | 171 +++++++++
{spark-tsfile => spark-iotdb-connector}/pom.xml | 27 +-
.../java/org/apache/iotdb/sparkdb/SQLConstant.java | 22 +-
.../scala/org/apache/iotdb/sparkdb/Converter.scala | 87 +++++
.../org/apache/iotdb/sparkdb/DefaultSource.scala | 42 ++
.../org/apache/iotdb/sparkdb/IoTDBOptions.scala | 42 ++
.../scala/org/apache/iotdb/sparkdb/IoTDBRDD.scala | 134 +++++++
.../org/apache/iotdb/sparkdb/IoTDBRelation.scala | 116 ++++++
.../org/apache/iotdb/sparkdb}/Transformer.scala | 33 +-
.../scala/org/apache/iotdb/sparkdb/package.scala | 35 ++
.../org/apache/iotdb/sparkdb/EnvironmentUtils.java | 282 ++++++++++++++
.../scala/org/apache/iotdb/sparkdb/IoTDBTest.scala | 115 ++++++
spark-tsfile/pom.xml | 14 +-
.../apache/iotdb/tsfile/io/TsFileRecordWriter.java | 1 +
.../java/org/apache/iotdb/tsfile/qp/Executor.java | 37 +-
.../org/apache/iotdb/tsfile/qp/QueryProcessor.java | 271 ++++++-------
.../iotdb/tsfile/qp/common/BasicOperator.java | 79 ++--
.../iotdb/tsfile/qp/common/FilterOperator.java | 244 ++++++------
.../apache/iotdb/tsfile/qp/common/Operator.java | 41 +-
.../apache/iotdb/tsfile/qp/common/SQLConstant.java | 249 ++++++------
.../apache/iotdb/tsfile/qp/common/SingleQuery.java | 53 +--
.../apache/iotdb/tsfile/qp/common/TSQueryPlan.java | 71 ++--
.../qp/exception/BasicOperatorException.java | 14 +-
.../tsfile/qp/exception/DNFOptimizeException.java | 14 +-
.../qp/exception/LogicalOptimizeException.java | 14 +-
.../tsfile/qp/exception/MergeFilterException.java | 14 +-
.../qp/exception/QueryOperatorException.java | 14 +-
.../qp/exception/QueryProcessorException.java | 30 +-
.../tsfile/qp/exception/RemoveNotException.java | 15 +-
.../tsfile/qp/optimizer/DNFFilterOptimizer.java | 241 ++++++------
.../tsfile/qp/optimizer/IFilterOptimizer.java | 10 +-
.../qp/optimizer/MergeSingleFilterOptimizer.java | 213 ++++++-----
.../tsfile/qp/optimizer/PhysicalOptimizer.java | 387 ++++++++++---------
.../tsfile/qp/optimizer/RemoveNotOptimizer.java | 144 +++----
.../scala/org/apache/iotdb/tsfile/Converter.scala | 14 +-
.../org/apache/iotdb/tsfile/DefaultSource.scala | 47 ++-
.../org/apache/iotdb/tsfile/NarrowConverter.scala | 138 ++++---
.../iotdb/tsfile/NarrowTsFileOutputWriter.scala | 2 +-
.../org/apache/iotdb/tsfile/Transformer.scala | 2 +-
.../org/apache/iotdb/tsfile/WideConverter.scala | 31 +-
.../iotdb/tsfile/WideTsFileOutputWriter.scala | 12 +-
.../scala/org/apache/iotdb/tsfile/package.scala | 9 +-
.../scala/org/apache/iotdb/tool/TsFileExample.java | 3 +-
.../org/apache/iotdb/tool/TsFileWriteTool.java | 55 +++
.../org/apache/iotdb/tsfile/ConverterTest.scala | 4 +-
.../org/apache/iotdb/tsfile/HDFSInputTest.java | 2 +-
.../scala/org/apache/iotdb/tsfile/TSFileSuit.scala | 423 ++++++++++++++++++++-
tsfile/pom.xml | 18 +
.../iotdb/tsfile/common/conf/TSFileConfig.java | 43 ++-
.../tsfile/common/constant/StatisticConstant.java | 2 +-
.../apache/iotdb/tsfile}/fileSystem/FSType.java | 2 +-
.../iotdb/tsfile/fileSystem/FileInputFactory.java | 51 +++
.../iotdb/tsfile/fileSystem/FileOutputFactory.java | 51 +++
.../apache/iotdb/tsfile/fileSystem/HDFSFile.java | 376 ++++++++++++++++++
.../apache/iotdb/tsfile/fileSystem}/HDFSInput.java | 13 +-
.../iotdb/tsfile/fileSystem}/HDFSOutput.java | 22 +-
.../iotdb/tsfile/fileSystem/TSFileFactory.java | 104 +++++
.../iotdb/tsfile/read/TsFileRestorableReader.java | 5 +-
.../iotdb/tsfile/read/TsFileSequenceReader.java | 11 +-
.../apache/iotdb/tsfile/write/TsFileWriter.java | 3 +
.../iotdb/tsfile/write/chunk/ChunkWriterImpl.java | 28 +-
.../write/writer/RestorableTsFileIOWriter.java | 3 +-
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 25 +-
.../tsfile/read/TsFileRestorableReaderTest.java | 5 +-
.../org/apache/iotdb/tsfile/utils/FileUtils.java | 3 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 15 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 33 +-
146 files changed, 5005 insertions(+), 2012 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
index 8df7cc8,fc11c68..e88794a
--- a/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/StorageEngine.java
@@@ -18,10 -18,8 +18,9 @@@
*/
package org.apache.iotdb.db.engine;
- import java.io.File;
import java.io.IOException;
import java.util.Collections;
+import java.util.ConcurrentModificationException;
import java.util.List;
import java.util.Map;
import java.util.Set;
diff --cc server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
index 9384d1b,3a5262d..9583b7f
--- a/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/merge/manage/MergeResource.java
@@@ -19,19 -19,6 +19,18 @@@
package org.apache.iotdb.db.engine.merge.manage;
+import static org.apache.iotdb.db.engine.merge.task.MergeTask.MERGE_SUFFIX;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
- import org.apache.iotdb.db.engine.fileSystem.FileFactory;
import org.apache.iotdb.db.engine.modification.Modification;
import org.apache.iotdb.db.engine.storagegroup.TsFileResource;
import org.apache.iotdb.db.query.reader.IPointReader;
diff --cc server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
index 54ebf8c,2cd8c87..dd23b87
--- a/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
+++ b/server/src/main/java/org/apache/iotdb/db/engine/storagegroup/StorageGroupProcessor.java
@@@ -24,11 -24,8 +24,10 @@@ import static org.apache.iotdb.tsfile.c
import java.io.File;
import java.io.IOException;
- import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.ConcurrentModificationException;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@@ -79,10 -73,12 +77,12 @@@ import org.apache.iotdb.tsfile.file.met
import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.file.metadata.enums.TSEncoding;
+ import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
+ import org.apache.iotdb.tsfile.fileSystem.TSFileFactory;
import org.apache.iotdb.tsfile.read.common.Path;
import org.apache.iotdb.tsfile.utils.Pair;
-import org.apache.iotdb.tsfile.write.schema.Schema;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
+import org.apache.iotdb.tsfile.write.schema.Schema;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@@ -224,7 -215,7 +224,8 @@@ public class StorageGroupProcessor
recoverUnseqFiles(unseqTsFiles);
String taskName = storageGroupName + "-" + System.currentTimeMillis();
- File mergingMods = new File(storageGroupSysDir, MERGING_MODIFICATION_FILE_NAME);
- File mergingMods = SystemFileFactory.INSTANCE.getFile(storageGroupSysDir, MERGING_MODIFICAITON_FILE_NAME);
++ File mergingMods = SystemFileFactory.INSTANCE.getFile(storageGroupSysDir,
++ MERGING_MODIFICATION_FILE_NAME);
if (mergingMods.exists()) {
mergingModification = new ModificationFile(mergingMods.getPath());
}
@@@ -246,10 -237,10 +247,10 @@@
}
}
- private List<TsFileResource> getAllFiles(List<String> folders) throws IOException {
+ private List<TsFileResource> getAllFiles(List<String> folders) {
List<File> tsFiles = new ArrayList<>();
for (String baseDir : folders) {
- File fileFolder = FileFactory.INSTANCE.getFile(baseDir, storageGroupName);
+ File fileFolder = TSFileFactory.INSTANCE.getFile(baseDir, storageGroupName);
if (!fileFolder.exists()) {
continue;
}
diff --cc server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
index 412544c,24abcc5..20f4fd7
--- a/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/metadata/MManager.java
@@@ -24,12 -24,11 +24,13 @@@ import java.util.concurrent.locks.Reent
import org.apache.iotdb.db.conf.IoTDBDescriptor;
import org.apache.iotdb.db.conf.adapter.IoTDBConfigDynamicAdapter;
+ import org.apache.iotdb.db.engine.fileSystem.SystemFileFactory;
import org.apache.iotdb.db.exception.ConfigAdjusterException;
import org.apache.iotdb.db.exception.MetadataErrorException;
+import org.apache.iotdb.db.exception.NotStorageGroupException;
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.monitor.MonitorConstants;
+import org.apache.iotdb.db.qp.logical.Operator.OperatorType;
import org.apache.iotdb.db.utils.RandomDeleteCache;
import org.apache.iotdb.tsfile.common.conf.TSFileConfig;
import org.apache.iotdb.tsfile.exception.cache.CacheException;
diff --cc server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
index 70fea4b,130362e..642d6e3
--- a/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/TSServiceImpl.java
@@@ -1058,7 -1088,7 +1077,7 @@@ public class TSServiceImpl implements T
}
@Override
- public TSRPCResp setStorageGroup(TSSetStorageGroupReq req) {
- public TSRPCResp setStorageGroup(String storageGroup) throws TException {
++ public TSRPCResp setStorageGroup(String storageGroup) {
if (!checkLogin()) {
logger.info(INFO_NOT_LOGIN, IoTDBConstant.GLOBAL_DB_NAME);
return new TSRPCResp(getStatus(TSStatusType.NOT_LOGIN_ERROR));