You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2019/12/16 13:34:37 UTC
[incubator-iotdb] 01/02: Merge branch 'master' into nvmlogging
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch nvmlogging
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit cb66b434f33b3abd3c482a58224aabc369f69c6d
Merge: fec0d2d df5dd1b
Author: mdf369 <95...@qq.com>
AuthorDate: Mon Dec 16 20:44:14 2019 +0800
Merge branch 'master' into nvmlogging
# Conflicts:
# server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
.travis.yml | 16 +-
README.md | 4 +-
client-py/compile.bat | 4 +-
client-py/compile.sh | 4 +-
client-py/readme.md | 2 +-
client-py/src/client_example.py | 133 +-
client/pom.xml | 2 +-
client/src/test/resources/logback.xml | 2 +-
distribution/pom.xml | 2 +-
docker/src/main/Dockerfile | 4 +-
docs/Development-Contributing.md | 6 +-
docs/Development-Document.md | 48 +
.../UserGuide/2-Concept/3-Encoding.md | 4 +-
.../UserGuide/2-Concept/4-Compression.md | 2 +-
.../UserGuide/4-Client/2-Programming - JDBC.md | 1 -
.../UserGuide/4-Client/3-Programming - Session.md | 2 +-
.../1-DDL (Data Definition Language).md | 4 +-
.../2-DML (Data Manipulation Language).md | 96 +-
.../5-Operation Manual/4-SQL Reference.md | 33 +-
.../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +-
.../6-System Tools/8-Load External Tsfile.md | 78 +
.../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +-
.../7-Ecosystem Integration/5-Hive TsFile.md | 8 +-
.../8-System Design (Developer)/1-Hierarchy.md | 8 +-
.../3-Writing Data on HDFS.md | 6 +-
.../0-Get Started/2-Frequently asked questions.md | 2 +-
.../UserGuide/2-Concept/3-Encoding.md | 2 -
.../UserGuide/2-Concept/4-Compression.md | 2 +-
.../UserGuide/3-Server/5-Docker Image.md | 2 +-
.../UserGuide/4-Client/2-Programming - JDBC.md | 3 +-
.../UserGuide/4-Client/3-Programming - Session.md | 14 +-
.../4-Client/5-Programming - TsFile API.md | 6 +-
.../2-DML (Data Manipulation Language).md | 73 +-
.../5-Operation Manual/4-SQL Reference.md | 32 +-
.../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +-
.../6-System Tools/8-Load External Tsfile.md | 72 +
.../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +-
.../7-Ecosystem Integration/3-Spark TsFile.md | 6 +-
.../7-Ecosystem Integration/4-Spark IoTDB.md | 8 +-
.../7-Ecosystem Integration/5-Hive TsFile.md | 10 +-
.../8-System Design (Developer)/1-Hierarchy.md | 8 +-
.../3-Writing Data on HDFS.md | 6 +-
example/hadoop/pom.xml | 6 +-
example/jdbc/pom.xml | 2 +-
.../main/java/org/apache/iotdb/JDBCExample.java | 7 +-
example/kafka/pom.xml | 4 +-
example/kafka/readme.md | 4 +-
example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
example/rocketmq/readme.md | 4 +-
example/session/pom.xml | 2 +-
.../main/java/org/apache/iotdb/SessionExample.java | 4 +-
example/tsfile/pom.xml | 2 +-
example/tsfile/readme.md | 2 +-
.../iotdb/tsfile/TsFileWriteWithTSRecord.java | 4 +-
grafana/conf/application.properties | 5 +-
grafana/pom.xml | 2 +-
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 39 +-
hadoop/README.md | 2 +-
hadoop/pom.xml | 4 +-
.../apache/iotdb/hadoop/tsfile/TSFHadoopTest.java | 3 +-
.../iotdb/hadoop/tsfile/constant/TestConstant.java | 18 +-
hadoop/src/test/resources/logback.xml | 2 +-
hive-connector/pom.xml | 12 +-
.../apache/iotdb/hive/TSFHiveInputFormatTest.java | 9 +-
.../apache/iotdb/hive/TSFHiveRecordReaderTest.java | 3 +-
.../apache/iotdb/hive/constant/TestConstant.java | 18 +-
jdbc/README.md | 3 +-
jdbc/pom.xml | 2 +-
.../main/java/org/apache/iotdb/jdbc/Constant.java | 3 -
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 150 +-
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 44 +-
.../iotdb/jdbc/IoTDBMetadataResultMetadata.java | 2 +-
.../apache/iotdb/jdbc/IoTDBMetadataResultSet.java | 5 -
.../jdbc/IoTDBPreparedInsertionStatement.java | 15 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 65 +-
.../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 321 ++--
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 125 +-
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 170 +-
.../test/java/org/apache/iotdb/jdbc/BatchTest.java | 34 +-
.../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 10 +-
.../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 4 +-
.../iotdb/jdbc/IoTDBPreparedStatementTest.java | 38 +-
.../apache/iotdb/jdbc/IoTDBQueryResultSetTest.java | 142 +-
.../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 35 +-
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 228 +--
pom.xml | 19 +-
server/pom.xml | 15 +-
.../resources/conf/iotdb-engine.properties | 2 +-
.../org/apache/iotdb/db/sql/parse/TqlLexer.g | 577 ------
.../org/apache/iotdb/db/sql/parse/TqlParser.g | 824 --------
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 862 +++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 27 +-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 19 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +-
.../iotdb/db/conf/adapter/CompressionRatio.java | 12 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 30 +-
.../engine/storagegroup/StorageGroupProcessor.java | 300 ++-
.../db/engine/storagegroup/TsFileResource.java | 8 +
.../exception/query/IllegalASTFormatException.java | 68 -
.../runtime/SQLParserException.java} | 30 +-
.../org/apache/iotdb/db/metadata/MManager.java | 23 +-
.../org/apache/iotdb/db/qp/QueryProcessor.java | 52 +-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 29 +-
.../iotdb/db/qp/constant/TqlParserConstant.java | 71 -
.../qp/executor/AbstractQueryProcessExecutor.java | 113 +-
.../db/qp/executor/IQueryProcessExecutor.java | 3 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 263 ++-
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../db/qp/logical/crud/BasicFunctionOperator.java | 5 +-
.../db/qp/logical/crud/BasicOperatorType.java | 5 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 70 +-
.../iotdb/db/qp/logical/sys/LoadFilesOperator.java | 67 +
.../sys/MoveFileOperator.java} | 33 +-
.../logical/sys/RemoveFileOperator.java} | 37 +-
.../iotdb/db/qp/physical/crud/GroupByPlan.java | 41 +-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 23 +
.../db/qp/physical/sys/LoadConfigurationPlan.java | 3 +-
.../sys/{ShowPlan.java => OperateFilePlan.java} | 56 +-
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 2 +-
.../iotdb/db/qp/strategy/LogicalGenerator.java | 1952 +++++++++----------
.../strategy/LogicalGeneratorError.java} | 29 +-
.../apache/iotdb/db/qp/strategy/ParseDriver.java | 76 +
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 41 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +-
.../iotdb/db/query/context/QueryContext.java | 10 +-
.../{JobFileManager.java => QueryFileManager.java} | 50 +-
.../db/query/control/QueryResourceManager.java | 35 +-
.../db/query/dataset/DeviceIterateDataSet.java | 24 +-
.../dataset/EngineDataSetWithValueFilter.java | 4 +-
.../dataset/EngineDataSetWithoutValueFilter.java | 13 +-
.../apache/iotdb/db/query/dataset/ListDataSet.java | 5 +-
.../{ListDataSet.java => SingleDataSet.java} | 25 +-
.../dataset/groupby/GroupByEngineDataSet.java | 67 +-
.../groupby/GroupByWithValueFilterDataSet.java | 30 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 19 +-
.../db/query/executor/AggregateEngineExecutor.java | 2 +-
.../iotdb/db/query/executor/EngineExecutor.java | 2 +-
.../iotdb/db/query/executor/EngineQueryRouter.java | 61 +-
.../db/query/executor/FillEngineExecutor.java | 5 +-
.../db/query/executor/IEngineQueryRouter.java | 15 +-
.../db/query/externalsort/ExternalSortJob.java | 12 +-
.../externalsort/ExternalSortJobScheduler.java | 10 +-
.../externalsort/SimpleExternalSortEngine.java | 2 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +-
.../resourceRelated/UnseqResourceMergeReader.java | 2 +-
.../UnseqResourceReaderByTimestamp.java | 2 +-
.../SeriesReaderWithoutValueFilter.java | 9 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 1 -
.../apache/iotdb/db/service/MetricsService.java | 290 +--
.../org/apache/iotdb/db/service/StaticResps.java | 89 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 792 ++++----
.../apache/iotdb/db/sql/parse/AstErrorNode.java | 58 -
.../org/apache/iotdb/db/sql/parse/AstNode.java | 362 ----
.../apache/iotdb/db/sql/parse/AstNodeOrigin.java | 110 --
.../java/org/apache/iotdb/db/sql/parse/Node.java | 42 -
.../org/apache/iotdb/db/sql/parse/ParseDriver.java | 192 --
.../org/apache/iotdb/db/sql/parse/ParseError.java | 52 -
.../org/apache/iotdb/db/sql/parse/ParseUtils.java | 43 -
.../iotdb/db/sync/receiver/load/FileLoader.java | 43 +-
.../{MemEst => memestimation}/MemEstTool.java | 2 +-
.../{MemEst => memestimation}/MemEstToolCmd.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 62 +
.../apache/iotdb/db/utils/QueryDataSetUtils.java | 90 +-
.../writelog/recover/TsFileRecoverPerformer.java | 6 +-
.../iotdb/db/auth/LocalFIleRoleAccessorTest.java | 3 +-
.../iotdb/db/auth/LocalFileRoleManagerTest.java | 3 +-
.../iotdb/db/auth/LocalFileUserAccessorTest.java | 3 +-
.../iotdb/db/auth/LocalFileUserManagerTest.java | 3 +-
.../strategy/DirectoryStrategyTest.java | 4 +-
.../org/apache/iotdb/db/constant/TestConstant.java | 15 +-
.../db/engine/cache/DeviceMetaDataCacheTest.java | 5 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 5 +-
.../apache/iotdb/db/engine/merge/MergeLogTest.java | 3 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 19 +-
.../iotdb/db/engine/merge/MergePerfTest.java | 3 +-
.../iotdb/db/engine/merge/MergeTaskTest.java | 3 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 13 +-
.../iotdb/db/engine/merge/MergeUpgradeTest.java | 13 +-
.../engine/modification/DeletionFileNodeTest.java | 4 +-
.../engine/modification/ModificationFileTest.java | 6 +-
.../io/LocalTextModificationAccessorTest.java | 6 +-
.../storagegroup/StorageGroupProcessorTest.java | 5 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 2 +-
.../engine/storagegroup/TsFileProcessorTest.java | 6 +-
.../version/SimpleFileVersionControllerTest.java | 3 +-
.../iotdb/db/integration/IOTDBGroupByIT.java | 269 +--
.../integration/IOTDBGroupByInnerIntervalIT.java | 296 +++
.../apache/iotdb/db/integration/IoTDBCloseIT.java | 12 +-
.../apache/iotdb/db/integration/IoTDBDaemonIT.java | 22 +-
.../iotdb/db/integration/IoTDBGroupbyDeviceIT.java | 28 +-
.../integration/IoTDBLoadExternalTsfileTest.java | 500 +++++
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 4 +-
.../db/integration/IoTDBMultiStatementsIT.java | 12 +-
.../iotdb/db/integration/IoTDBPreparedStmtIT.java | 2 -
.../iotdb/db/integration/IoTDBQueryDemoIT.java | 94 +-
.../db/integration/IoTDBSequenceDataQueryIT.java | 12 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 16 +-
.../iotdb/db/metadata/MManagerImproveTest.java | 2 +-
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 16 +-
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 237 ++-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 95 +-
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 7 +-
.../iotdb/db/qp/plan/TestConcatOptimizer.java | 5 +-
.../iotdb/db/qp/strategy/LogicalGeneratorTest.java | 5 +-
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 19 +-
.../db/query/control/FileReaderManagerTest.java | 9 +-
.../query/executor/GroupByEngineDataSetTest.java | 117 +-
.../query/externalsort/ExternalSortEngineTest.java | 6 +-
...ExternalSortFileSerializerDeserializerTest.java | 3 +-
.../iotdb/db/query/reader/ReaderTestHelper.java | 3 +-
.../org/apache/iotdb/db/sql/TqlParserTest.java | 2021 --------------------
.../apache/iotdb/db/tools/IoTDBWatermarkTest.java | 5 +-
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 9 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 15 +-
.../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 3 +-
server/src/test/resources/iotdb-engine.properties | 21 +
server/src/test/resources/logback.xml | 2 +-
service-rpc/pom.xml | 50 +-
service-rpc/rpc-changelist.md | 26 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 -
service-rpc/src/main/thrift/rpc.thrift | 185 +-
service-rpc/src/pypi/README.md | 63 +
service-rpc/src/pypi/setup.py | 55 +
session/pom.xml | 2 +-
.../java/org/apache/iotdb/session/Session.java | 113 +-
.../org/apache/iotdb/session/SessionDataSet.java | 159 +-
.../org/apache/iotdb/session/SessionUtils.java | 78 +-
.../org/apache/iotdb/session/IoTDBSessionIT.java | 84 +-
.../iotdb/session/utils/EnvironmentUtils.java | 20 +-
session/src/test/resources/iotdb-engine.properties | 20 +
spark-iotdb-connector/Readme.md | 8 +-
spark-iotdb-connector/pom.xml | 2 +-
.../src/test/resources/iotdb-engine.properties | 21 +
.../apache/iotdb/spark/db/EnvironmentUtils.java | 13 +-
.../org/apache/iotdb/spark/db/IoTDBTest.scala | 2 +
spark-tsfile/README.md | 6 +-
spark-tsfile/pom.xml | 4 +-
.../apache/iotdb/spark/constant/TestConstant.java | 17 +-
.../apache/iotdb/spark/tsfile/ConverterTest.scala | 3 +-
.../apache/iotdb/spark/tsfile/HDFSInputTest.java | 4 +-
.../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 3 +-
tsfile/README.md | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 6 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 101 +
.../tsfile/read/filter/factory/FilterType.java | 2 +-
.../query/dataset/DataSetWithTimeGenerator.java | 5 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 59 +-
...meseriesTestConstant.java => TestConstant.java} | 14 +-
.../iotdb/tsfile/file/header/PageHeaderTest.java | 6 +-
.../file/metadata/ChunkGroupMetaDataTest.java | 3 +-
.../file/metadata/TimeSeriesMetadataTest.java | 4 +-
.../file/metadata/TsDeviceMetadataIndexTest.java | 4 +-
.../tsfile/file/metadata/TsFileMetaDataTest.java | 202 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 750 ++++----
.../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 733 +++----
.../tsfile/read/filter/GroupByFilterTest.java | 126 ++
.../TsFileGeneratorForSeriesReaderByTimestamp.java | 451 ++---
.../apache/iotdb/tsfile/utils/BytesUtilsTest.java | 22 +-
.../apache/iotdb/tsfile/utils/FileGenerator.java | 429 ++---
.../apache/iotdb/tsfile/utils/FileUtilsTest.java | 10 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 7 +-
.../org/apache/iotdb/tsfile/write/PerfTest.java | 7 +-
.../iotdb/tsfile/write/ReadPageInMemTest.java | 3 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 3 +-
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 3 +-
.../org/apache/iotdb/tsfile/write/WriteTest.java | 7 +-
.../iotdb/tsfile/write/writer/PageWriterTest.java | 4 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 4 +-
tsfile/src/test/resources/logback.xml | 2 +-
272 files changed, 8641 insertions(+), 9986 deletions(-)
diff --cc server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 8feb826,f71b8f3..eaee4bb
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@@ -68,9 -69,8 +70,9 @@@ public class EnvironmentUtils
private static long oldGroupSizeInByte = config.getMemtableSizeThreshold();
public static void cleanEnv() throws IOException, StorageEngineException {
+ NVMSpaceManager.close();
- QueryResourceManager.getInstance().endQueryForGivenJob(TEST_QUERY_JOB_ID);
+ QueryResourceManager.getInstance().endQuery(TEST_QUERY_JOB_ID);
// clear opened file streams
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
@@@ -145,8 -141,8 +145,8 @@@
/**
* disable memory control</br> this function should be called before all code in the setup
*/
- public static void envSetUp() throws StartupException, IOException {
+ public static void envSetUp() throws StartupException {
- IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(false);
+ config.setEnableParameterAdapter(false);
MManager.getInstance().init();
IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
@@@ -168,10 -164,8 +168,10 @@@
MultiFileLogNodeManager.getInstance().start();
FlushManager.getInstance().start();
MergeManager.getINSTANCE().start();
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignJobId();
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId();
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
+
+ NVMSpaceManager.init(config.getNvmDir());
}
private static void createAllDir() {
@@@ -187,10 -181,9 +187,11 @@@
createDir(config.getSystemDir());
// create wal
createDir(config.getWalFolder());
+ // create index
+ createDir(config.getIndexFileDir());
// create query
createDir(config.getQueryDir());
+ createDir(TestConstant.OUTPUT_DATA_DIR);
// create data
for (String dataDir : config.getDataDirs()) {
createDir(dataDir);