You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by ea...@apache.org on 2019/12/16 13:34:36 UTC
[incubator-iotdb] branch nvmlogging updated (fec0d2d -> c4fd53c)
This is an automated email from the ASF dual-hosted git repository.
east pushed a change to branch nvmlogging
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from fec0d2d update
add 49af5b3 [IOTDB-311]Generate anything in test to the target directory. (#578)
add 8c7b51a Bump jetty.version from 9.4.22.v20191022 to 9.4.24.v20191120 (#613)
add 927239b [IOTDB-190] upgrade from antlr3 to antlr4 (#564)
add 8d0dd60 [IOTDB-329] add stop MetricService for all EnvironmentUtils (#615)
add fb8fa2f remove some unnecessary codes (#616)
add 30b0a66 remove createNewDataSet unnecessary parameter
add 6e968b6 change fetch_size to fetchSize in rpc
add cbf73d0 remove comments
add be7c91f rename resources in TSServiceImpl
add 036968e update version (#618)
add 07cb959 [IOTDB-336]declare that the release version of JDK is 8 (#617)
add 5f60ff4 add timestamp precision properties (#620)
add f333953 Remove unnecessary call method of TSStatus (#622)
add c7fad40 IOTDB-334 Move LIMIT&OFFSET to Server Side (#621)
add 0438f91 fix data type bug in grafana (#625)
add 571043f remove a constructor in SeriesReaderWithoutValueFilter
add 2f2f72d [IOTDB-217] A new GROUPBY syntax with supporting inner interval (#570)
add 7a4adda Fix TTL doc and add some description
add 3b38e56 Fix jdbc example for groupby (#628)
add 43616c5 [IOTDB-287] Allow anything in a JDBC URL. (#624)
add 50494e3 modify antlr grammar to support load/remove/move path
add 734c217 add remove tsfile and move tsfile to target directory
add b68c549 complete load tsfile and fix some bugs of corner cases
add 83db91a Merge branch 'master' into add_load_tsfile_feature
add ad864ad add unit test
add 41bf8ea add unit test
add a9e329c add load external tsfile doc
add 2b9b7ec fix a bug of ut
add 67cc066 upgrade to anltr4
add 741908c add auto create schema
add 498e803 merge master
add bd312cd add ut for load files with creating metadata
add f4b4b80 fix ut bug and update doc
add 1c8aa34 Merge pull request #597 from apache/add_load_tsfile_feature
add 87f54a4 [IOTDB-343] Add test method in session (#629)
add 4325ae4 fix a bug of loading external tsfile test
add 57488f2 fix a bug of re-load the same external tsfile will lead to read-only mode
add 4a60924 Python: Support pypi distribution for Python Client (#623)
add 036a9fa Change the return content in TSQueryDataSet and remove the row record in IoTDBQueryResultSet (#626)
add 83cbb32 Merge branch 'master' of github.com:apache/incubator-iotdb
add 0771b63 [IOTDB-347] Remove the confusing limit in IoTDBQueryResultSetTest.java (#633)
add 919cbca Add a new page for brief introduction of document contributing (#632)
add 0212fd8 Move show version to server-side (#630)
add f0a4296 Bug fix for document encoding (#634)
add b70fcc5 Python example next (#636)
add 883f98a Add antlr error listener. (#635)
add 9a59c60 upgrade thrift from 0.12 to 0.13 (#637)
add 5a11bf5 fix session example
add 025653a try to fix the batch error. (#639)
add b0fd6b1 fix bug of dead lock in compression ratio (#641)
add 901012a fix a bug of renaming tsfile while it has naming conflicts (#640)
add 597da57 fix null pointer exception in tsfile recover performer
add 122fba0 Merge pull request #642 from apache/fix_jira_357
add f951bd3 [IOTDB-306] new reader with ChunkMetaData for aggregateFunction (#614)
add 8bf3401 Revert "[IOTDB-306] new reader with ChunkMetaData for aggregateFunction (#614)" (#644)
add 06654ab [IOTDB-363] Correct the doc link error for `Development-Contributing.md` and `Development-Document.md`.
add c50c73f Correct typo in Development-Contributing.md. (#649)
add f18b9ad [IOTDB-365] Improve the link of confluence in Development-Document.md (#651)
add df5dd1b [IOTDB-361]Refactor session management (#650)
new cb66b43 Merge branch 'master' into nvmlogging
new c4fd53c update
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.travis.yml | 16 +-
README.md | 4 +-
client-py/compile.bat | 4 +-
client-py/compile.sh | 4 +-
client-py/readme.md | 2 +-
client-py/src/client_example.py | 133 +-
client/pom.xml | 2 +-
client/src/test/resources/logback.xml | 2 +-
distribution/pom.xml | 2 +-
docker/src/main/Dockerfile | 4 +-
docs/Development-Contributing.md | 6 +-
docs/Development-Document.md | 48 +
.../UserGuide/2-Concept/3-Encoding.md | 4 +-
.../UserGuide/2-Concept/4-Compression.md | 2 +-
.../UserGuide/4-Client/2-Programming - JDBC.md | 1 -
.../UserGuide/4-Client/3-Programming - Session.md | 2 +-
.../1-DDL (Data Definition Language).md | 4 +-
.../2-DML (Data Manipulation Language).md | 96 +-
.../5-Operation Manual/4-SQL Reference.md | 33 +-
.../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +-
.../6-System Tools/8-Load External Tsfile.md | 78 +
.../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +-
.../7-Ecosystem Integration/5-Hive TsFile.md | 8 +-
.../8-System Design (Developer)/1-Hierarchy.md | 8 +-
.../3-Writing Data on HDFS.md | 6 +-
.../0-Get Started/2-Frequently asked questions.md | 2 +-
.../UserGuide/2-Concept/3-Encoding.md | 2 -
.../UserGuide/2-Concept/4-Compression.md | 2 +-
.../UserGuide/3-Server/5-Docker Image.md | 2 +-
.../UserGuide/4-Client/2-Programming - JDBC.md | 3 +-
.../UserGuide/4-Client/3-Programming - Session.md | 14 +-
.../4-Client/5-Programming - TsFile API.md | 6 +-
.../2-DML (Data Manipulation Language).md | 73 +-
.../5-Operation Manual/4-SQL Reference.md | 32 +-
.../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +-
.../6-System Tools/8-Load External Tsfile.md | 72 +
.../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +-
.../7-Ecosystem Integration/3-Spark TsFile.md | 6 +-
.../7-Ecosystem Integration/4-Spark IoTDB.md | 8 +-
.../7-Ecosystem Integration/5-Hive TsFile.md | 10 +-
.../8-System Design (Developer)/1-Hierarchy.md | 8 +-
.../3-Writing Data on HDFS.md | 6 +-
example/hadoop/pom.xml | 6 +-
example/jdbc/pom.xml | 2 +-
.../main/java/org/apache/iotdb/JDBCExample.java | 7 +-
example/kafka/pom.xml | 4 +-
example/kafka/readme.md | 4 +-
example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
example/rocketmq/readme.md | 4 +-
example/session/pom.xml | 2 +-
.../main/java/org/apache/iotdb/SessionExample.java | 4 +-
example/tsfile/pom.xml | 2 +-
example/tsfile/readme.md | 2 +-
.../iotdb/tsfile/TsFileWriteWithTSRecord.java | 4 +-
grafana/conf/application.properties | 5 +-
grafana/pom.xml | 2 +-
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 39 +-
hadoop/README.md | 2 +-
hadoop/pom.xml | 4 +-
.../apache/iotdb/hadoop/tsfile/TSFHadoopTest.java | 3 +-
.../iotdb/hadoop/tsfile/constant/TestConstant.java | 12 +-
hadoop/src/test/resources/logback.xml | 2 +-
hive-connector/pom.xml | 12 +-
.../apache/iotdb/hive/TSFHiveInputFormatTest.java | 9 +-
.../apache/iotdb/hive/TSFHiveRecordReaderTest.java | 3 +-
.../apache/iotdb/hive/constant/TestConstant.java | 12 +-
jdbc/README.md | 3 +-
jdbc/pom.xml | 2 +-
.../main/java/org/apache/iotdb/jdbc/Constant.java | 3 -
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 150 +-
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 44 +-
.../iotdb/jdbc/IoTDBMetadataResultMetadata.java | 2 +-
.../apache/iotdb/jdbc/IoTDBMetadataResultSet.java | 5 -
.../jdbc/IoTDBPreparedInsertionStatement.java | 15 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 65 +-
.../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 321 ++--
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 125 +-
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 170 +-
.../test/java/org/apache/iotdb/jdbc/BatchTest.java | 34 +-
.../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 10 +-
.../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 4 +-
.../iotdb/jdbc/IoTDBPreparedStatementTest.java | 38 +-
.../apache/iotdb/jdbc/IoTDBQueryResultSetTest.java | 142 +-
.../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 35 +-
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 228 +--
pom.xml | 19 +-
server/pom.xml | 15 +-
.../resources/conf/iotdb-engine.properties | 2 +-
.../org/apache/iotdb/db/sql/parse/TqlLexer.g | 577 ------
.../org/apache/iotdb/db/sql/parse/TqlParser.g | 824 --------
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 862 +++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 27 +-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 19 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +-
.../iotdb/db/conf/adapter/CompressionRatio.java | 12 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 30 +-
.../engine/storagegroup/StorageGroupProcessor.java | 300 ++-
.../db/engine/storagegroup/TsFileResource.java | 8 +
.../exception/query/IllegalASTFormatException.java | 68 -
...ilureException.java => SQLParserException.java} | 18 +-
.../org/apache/iotdb/db/metadata/MManager.java | 23 +-
.../{NVMLongTVList.java => NVMBooleanTVList.java} | 43 +-
.../{NVMIntTVList.java => NVMDoubleTVList.java} | 42 +-
.../{NVMIntTVList.java => NVMFloatTVList.java} | 42 +-
.../iotdb/db/nvm/datastructure/NVMTVList.java | 6 +-
.../iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java | 2 +-
.../apache/iotdb/db/nvm/space/NVMSpaceManager.java | 42 +-
.../org/apache/iotdb/db/qp/QueryProcessor.java | 52 +-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 29 +-
.../iotdb/db/qp/constant/TqlParserConstant.java | 71 -
.../qp/executor/AbstractQueryProcessExecutor.java | 113 +-
.../db/qp/executor/IQueryProcessExecutor.java | 3 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 263 ++-
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../db/qp/logical/crud/BasicFunctionOperator.java | 5 +-
.../db/qp/logical/crud/BasicOperatorType.java | 5 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 70 +-
...ataAuthOperator.java => LoadFilesOperator.java} | 50 +-
...{ShowTTLOperator.java => MoveFileOperator.java} | 27 +-
...howTTLOperator.java => RemoveFileOperator.java} | 26 +-
.../iotdb/db/qp/physical/crud/GroupByPlan.java | 41 +-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 23 +
.../db/qp/physical/sys/LoadConfigurationPlan.java | 3 +-
.../sys/{ShowPlan.java => OperateFilePlan.java} | 56 +-
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 2 +-
.../iotdb/db/qp/strategy/LogicalGenerator.java | 1952 +++++++++----------
.../strategy/LogicalGeneratorError.java} | 18 +-
.../apache/iotdb/db/qp/strategy/ParseDriver.java | 76 +
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 41 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +-
.../iotdb/db/query/context/QueryContext.java | 10 +-
.../{JobFileManager.java => QueryFileManager.java} | 50 +-
.../db/query/control/QueryResourceManager.java | 35 +-
.../db/query/dataset/DeviceIterateDataSet.java | 24 +-
.../dataset/EngineDataSetWithValueFilter.java | 4 +-
.../dataset/EngineDataSetWithoutValueFilter.java | 13 +-
.../apache/iotdb/db/query/dataset/ListDataSet.java | 5 +-
.../{ListDataSet.java => SingleDataSet.java} | 25 +-
.../dataset/groupby/GroupByEngineDataSet.java | 67 +-
.../groupby/GroupByWithValueFilterDataSet.java | 30 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 19 +-
.../db/query/executor/AggregateEngineExecutor.java | 2 +-
.../iotdb/db/query/executor/EngineExecutor.java | 2 +-
.../iotdb/db/query/executor/EngineQueryRouter.java | 61 +-
.../db/query/executor/FillEngineExecutor.java | 5 +-
.../db/query/executor/IEngineQueryRouter.java | 15 +-
.../db/query/externalsort/ExternalSortJob.java | 12 +-
.../externalsort/ExternalSortJobScheduler.java | 10 +-
.../externalsort/SimpleExternalSortEngine.java | 2 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +-
.../resourceRelated/UnseqResourceMergeReader.java | 2 +-
.../UnseqResourceReaderByTimestamp.java | 2 +-
.../SeriesReaderWithoutValueFilter.java | 9 +-
.../apache/iotdb/db/rescon/TVListAllocator.java | 21 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 4 +-
.../apache/iotdb/db/service/MetricsService.java | 290 +--
.../org/apache/iotdb/db/service/StaticResps.java | 89 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 792 ++++----
.../org/apache/iotdb/db/sql/ParseGenerator.java | 43 -
.../apache/iotdb/db/sql/parse/AstErrorNode.java | 58 -
.../org/apache/iotdb/db/sql/parse/AstNode.java | 362 ----
.../apache/iotdb/db/sql/parse/AstNodeOrigin.java | 110 --
.../java/org/apache/iotdb/db/sql/parse/Node.java | 42 -
.../org/apache/iotdb/db/sql/parse/ParseDriver.java | 192 --
.../org/apache/iotdb/db/sql/parse/ParseError.java | 52 -
.../apache/iotdb/db/sql/parse/ParseException.java | 48 -
.../org/apache/iotdb/db/sql/parse/ParseUtils.java | 43 -
.../iotdb/db/sync/receiver/load/FileLoader.java | 43 +-
.../{MemEst => memestimation}/MemEstTool.java | 2 +-
.../{MemEst => memestimation}/MemEstToolCmd.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 62 +
.../apache/iotdb/db/utils/QueryDataSetUtils.java | 90 +-
.../writelog/recover/TsFileRecoverPerformer.java | 6 +-
.../iotdb/db/auth/LocalFIleRoleAccessorTest.java | 3 +-
.../iotdb/db/auth/LocalFileRoleManagerTest.java | 3 +-
.../iotdb/db/auth/LocalFileUserAccessorTest.java | 3 +-
.../iotdb/db/auth/LocalFileUserManagerTest.java | 3 +-
.../strategy/DirectoryStrategyTest.java | 4 +-
.../org/apache/iotdb/db/constant/TestConstant.java | 13 +-
.../db/engine/cache/DeviceMetaDataCacheTest.java | 5 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 5 +-
.../apache/iotdb/db/engine/merge/MergeLogTest.java | 3 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 19 +-
.../iotdb/db/engine/merge/MergePerfTest.java | 3 +-
.../iotdb/db/engine/merge/MergeTaskTest.java | 3 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 13 +-
.../iotdb/db/engine/merge/MergeUpgradeTest.java | 13 +-
.../engine/modification/DeletionFileNodeTest.java | 4 +-
.../engine/modification/ModificationFileTest.java | 6 +-
.../io/LocalTextModificationAccessorTest.java | 6 +-
.../storagegroup/StorageGroupProcessorTest.java | 5 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 2 +-
.../engine/storagegroup/TsFileProcessorTest.java | 6 +-
.../version/SimpleFileVersionControllerTest.java | 3 +-
.../iotdb/db/integration/IOTDBGroupByIT.java | 269 +--
.../integration/IOTDBGroupByInnerIntervalIT.java | 296 +++
.../apache/iotdb/db/integration/IoTDBCloseIT.java | 12 +-
.../apache/iotdb/db/integration/IoTDBDaemonIT.java | 22 +-
.../iotdb/db/integration/IoTDBGroupbyDeviceIT.java | 28 +-
.../integration/IoTDBLoadExternalTsfileTest.java | 500 +++++
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 4 +-
.../db/integration/IoTDBMultiStatementsIT.java | 12 +-
.../iotdb/db/integration/IoTDBPreparedStmtIT.java | 2 -
.../iotdb/db/integration/IoTDBQueryDemoIT.java | 94 +-
.../db/integration/IoTDBSequenceDataQueryIT.java | 12 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 16 +-
.../iotdb/db/metadata/MManagerImproveTest.java | 2 +-
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 16 +-
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 237 ++-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 95 +-
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 7 +-
.../iotdb/db/qp/plan/TestConcatOptimizer.java | 5 +-
.../iotdb/db/qp/strategy/LogicalGeneratorTest.java | 5 +-
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 19 +-
.../db/query/control/FileReaderManagerTest.java | 9 +-
.../query/executor/GroupByEngineDataSetTest.java | 117 +-
.../query/externalsort/ExternalSortEngineTest.java | 6 +-
...ExternalSortFileSerializerDeserializerTest.java | 3 +-
.../iotdb/db/query/reader/ReaderTestHelper.java | 3 +-
.../org/apache/iotdb/db/sql/TqlParserTest.java | 2021 --------------------
.../apache/iotdb/db/tools/IoTDBWatermarkTest.java | 5 +-
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 9 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 21 +-
.../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 3 +-
server/src/test/resources/iotdb-engine.properties | 21 +
server/src/test/resources/logback.xml | 2 +-
service-rpc/pom.xml | 50 +-
service-rpc/rpc-changelist.md | 26 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 -
service-rpc/src/main/thrift/rpc.thrift | 185 +-
service-rpc/src/pypi/README.md | 63 +
service-rpc/src/pypi/setup.py | 55 +
session/pom.xml | 2 +-
.../java/org/apache/iotdb/session/Session.java | 113 +-
.../org/apache/iotdb/session/SessionDataSet.java | 159 +-
.../org/apache/iotdb/session/SessionUtils.java | 78 +-
.../org/apache/iotdb/session/IoTDBSessionIT.java | 84 +-
.../iotdb/session/utils/EnvironmentUtils.java | 20 +-
session/src/test/resources/iotdb-engine.properties | 20 +
spark-iotdb-connector/Readme.md | 8 +-
spark-iotdb-connector/pom.xml | 2 +-
.../src/test/resources/iotdb-engine.properties | 21 +
.../apache/iotdb/spark/db/EnvironmentUtils.java | 13 +-
.../org/apache/iotdb/spark/db/IoTDBTest.scala | 2 +
spark-tsfile/README.md | 6 +-
spark-tsfile/pom.xml | 4 +-
.../apache/iotdb/spark/constant/TestConstant.java | 9 +-
.../apache/iotdb/spark/tsfile/ConverterTest.scala | 3 +-
.../apache/iotdb/spark/tsfile/HDFSInputTest.java | 4 +-
.../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 3 +-
tsfile/README.md | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 6 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 101 +
.../tsfile/read/filter/factory/FilterType.java | 2 +-
.../query/dataset/DataSetWithTimeGenerator.java | 5 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 59 +-
...meseriesTestConstant.java => TestConstant.java} | 14 +-
.../iotdb/tsfile/file/header/PageHeaderTest.java | 6 +-
.../file/metadata/ChunkGroupMetaDataTest.java | 3 +-
.../file/metadata/TimeSeriesMetadataTest.java | 4 +-
.../file/metadata/TsDeviceMetadataIndexTest.java | 4 +-
.../tsfile/file/metadata/TsFileMetaDataTest.java | 202 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 750 ++++----
.../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 733 +++----
.../tsfile/read/filter/GroupByFilterTest.java | 126 ++
.../TsFileGeneratorForSeriesReaderByTimestamp.java | 451 ++---
.../apache/iotdb/tsfile/utils/BytesUtilsTest.java | 22 +-
.../iotdb/tsfile/utils/CommonTestConstant.java | 31 -
.../apache/iotdb/tsfile/utils/FileGenerator.java | 429 ++---
.../apache/iotdb/tsfile/utils/FileUtilsTest.java | 10 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 7 +-
.../org/apache/iotdb/tsfile/write/PerfTest.java | 7 +-
.../iotdb/tsfile/write/ReadPageInMemTest.java | 3 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 3 +-
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 3 +-
.../org/apache/iotdb/tsfile/write/WriteTest.java | 7 +-
.../iotdb/tsfile/write/writer/PageWriterTest.java | 4 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 4 +-
tsfile/src/test/resources/logback.xml | 2 +-
282 files changed, 8720 insertions(+), 10157 deletions(-)
create mode 100644 docs/Development-Document.md
create mode 100644 docs/Documentation-CHN/UserGuide/6-System Tools/8-Load External Tsfile.md
create mode 100644 docs/Documentation/UserGuide/6-System Tools/8-Load External Tsfile.md
copy client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java => hadoop/src/test/java/org/apache/iotdb/hadoop/tsfile/constant/TestConstant.java (74%)
copy client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java => hive-connector/src/test/java/org/apache/iotdb/hive/constant/TestConstant.java (74%)
delete mode 100644 server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TqlLexer.g
delete mode 100644 server/src/main/antlr3/org/apache/iotdb/db/sql/parse/TqlParser.g
create mode 100644 server/src/main/antlr4/org/apache/iotdb/db/qp/strategy/SqlBase.g4
delete mode 100644 server/src/main/java/org/apache/iotdb/db/exception/query/IllegalASTFormatException.java
copy server/src/main/java/org/apache/iotdb/db/exception/runtime/{StorageEngineFailureException.java => SQLParserException.java} (65%)
copy server/src/main/java/org/apache/iotdb/db/nvm/datastructure/{NVMLongTVList.java => NVMBooleanTVList.java} (82%)
copy server/src/main/java/org/apache/iotdb/db/nvm/datastructure/{NVMIntTVList.java => NVMDoubleTVList.java} (82%)
copy server/src/main/java/org/apache/iotdb/db/nvm/datastructure/{NVMIntTVList.java => NVMFloatTVList.java} (83%)
delete mode 100644 server/src/main/java/org/apache/iotdb/db/qp/constant/TqlParserConstant.java
copy server/src/main/java/org/apache/iotdb/db/qp/logical/sys/{DataAuthOperator.java => LoadFilesOperator.java} (50%)
copy server/src/main/java/org/apache/iotdb/db/qp/logical/sys/{ShowTTLOperator.java => MoveFileOperator.java} (69%)
copy server/src/main/java/org/apache/iotdb/db/qp/logical/sys/{ShowTTLOperator.java => RemoveFileOperator.java} (70%)
copy server/src/main/java/org/apache/iotdb/db/qp/physical/sys/{ShowPlan.java => OperateFilePlan.java} (50%)
copy server/src/main/java/org/apache/iotdb/db/{concurrent/IoTDBDefaultThreadExceptionHandler.java => qp/strategy/LogicalGeneratorError.java} (56%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/qp/strategy/ParseDriver.java
rename server/src/main/java/org/apache/iotdb/db/query/control/{JobFileManager.java => QueryFileManager.java} (66%)
copy server/src/main/java/org/apache/iotdb/db/query/dataset/{ListDataSet.java => SingleDataSet.java} (75%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/service/StaticResps.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/ParseGenerator.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/AstErrorNode.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/AstNode.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/AstNodeOrigin.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/Node.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/ParseDriver.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/ParseError.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/ParseException.java
delete mode 100644 server/src/main/java/org/apache/iotdb/db/sql/parse/ParseUtils.java
rename server/src/main/java/org/apache/iotdb/db/tools/{MemEst => memestimation}/MemEstTool.java (98%)
rename server/src/main/java/org/apache/iotdb/db/tools/{MemEst => memestimation}/MemEstToolCmd.java (98%)
create mode 100644 server/src/main/java/org/apache/iotdb/db/utils/FileLoaderUtils.java
copy client/src/main/java/org/apache/iotdb/exception/ArgsErrorException.java => server/src/test/java/org/apache/iotdb/db/constant/TestConstant.java (73%)
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IOTDBGroupByInnerIntervalIT.java
create mode 100644 server/src/test/java/org/apache/iotdb/db/integration/IoTDBLoadExternalTsfileTest.java
delete mode 100644 server/src/test/java/org/apache/iotdb/db/sql/TqlParserTest.java
create mode 100644 server/src/test/resources/iotdb-engine.properties
create mode 100644 service-rpc/src/pypi/README.md
create mode 100644 service-rpc/src/pypi/setup.py
create mode 100644 session/src/test/resources/iotdb-engine.properties
create mode 100644 spark-iotdb-connector/src/test/resources/iotdb-engine.properties
copy server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocatorMBean.java => spark-tsfile/src/test/scala/org/apache/iotdb/spark/constant/TestConstant.java (80%)
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/read/filter/GroupByFilter.java
rename tsfile/src/test/java/org/apache/iotdb/tsfile/constant/{TimeseriesTestConstant.java => TestConstant.java} (72%)
mode change 100755 => 100644
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/read/filter/GroupByFilterTest.java
delete mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/utils/CommonTestConstant.java
[incubator-iotdb] 01/02: Merge branch 'master' into nvmlogging
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch nvmlogging
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit cb66b434f33b3abd3c482a58224aabc369f69c6d
Merge: fec0d2d df5dd1b
Author: mdf369 <95...@qq.com>
AuthorDate: Mon Dec 16 20:44:14 2019 +0800
Merge branch 'master' into nvmlogging
# Conflicts:
# server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
.travis.yml | 16 +-
README.md | 4 +-
client-py/compile.bat | 4 +-
client-py/compile.sh | 4 +-
client-py/readme.md | 2 +-
client-py/src/client_example.py | 133 +-
client/pom.xml | 2 +-
client/src/test/resources/logback.xml | 2 +-
distribution/pom.xml | 2 +-
docker/src/main/Dockerfile | 4 +-
docs/Development-Contributing.md | 6 +-
docs/Development-Document.md | 48 +
.../UserGuide/2-Concept/3-Encoding.md | 4 +-
.../UserGuide/2-Concept/4-Compression.md | 2 +-
.../UserGuide/4-Client/2-Programming - JDBC.md | 1 -
.../UserGuide/4-Client/3-Programming - Session.md | 2 +-
.../1-DDL (Data Definition Language).md | 4 +-
.../2-DML (Data Manipulation Language).md | 96 +-
.../5-Operation Manual/4-SQL Reference.md | 33 +-
.../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +-
.../6-System Tools/8-Load External Tsfile.md | 78 +
.../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +-
.../7-Ecosystem Integration/5-Hive TsFile.md | 8 +-
.../8-System Design (Developer)/1-Hierarchy.md | 8 +-
.../3-Writing Data on HDFS.md | 6 +-
.../0-Get Started/2-Frequently asked questions.md | 2 +-
.../UserGuide/2-Concept/3-Encoding.md | 2 -
.../UserGuide/2-Concept/4-Compression.md | 2 +-
.../UserGuide/3-Server/5-Docker Image.md | 2 +-
.../UserGuide/4-Client/2-Programming - JDBC.md | 3 +-
.../UserGuide/4-Client/3-Programming - Session.md | 14 +-
.../4-Client/5-Programming - TsFile API.md | 6 +-
.../2-DML (Data Manipulation Language).md | 73 +-
.../5-Operation Manual/4-SQL Reference.md | 32 +-
.../UserGuide/6-System Tools/5-Log Visualizer.md | 2 +-
.../6-System Tools/8-Load External Tsfile.md | 72 +
.../7-Ecosystem Integration/2-MapReduce TsFile.md | 2 +-
.../7-Ecosystem Integration/3-Spark TsFile.md | 6 +-
.../7-Ecosystem Integration/4-Spark IoTDB.md | 8 +-
.../7-Ecosystem Integration/5-Hive TsFile.md | 10 +-
.../8-System Design (Developer)/1-Hierarchy.md | 8 +-
.../3-Writing Data on HDFS.md | 6 +-
example/hadoop/pom.xml | 6 +-
example/jdbc/pom.xml | 2 +-
.../main/java/org/apache/iotdb/JDBCExample.java | 7 +-
example/kafka/pom.xml | 4 +-
example/kafka/readme.md | 4 +-
example/pom.xml | 2 +-
example/rocketmq/pom.xml | 2 +-
example/rocketmq/readme.md | 4 +-
example/session/pom.xml | 2 +-
.../main/java/org/apache/iotdb/SessionExample.java | 4 +-
example/tsfile/pom.xml | 2 +-
example/tsfile/readme.md | 2 +-
.../iotdb/tsfile/TsFileWriteWithTSRecord.java | 4 +-
grafana/conf/application.properties | 5 +-
grafana/pom.xml | 2 +-
.../iotdb/web/grafana/dao/impl/BasicDaoImpl.java | 39 +-
hadoop/README.md | 2 +-
hadoop/pom.xml | 4 +-
.../apache/iotdb/hadoop/tsfile/TSFHadoopTest.java | 3 +-
.../iotdb/hadoop/tsfile/constant/TestConstant.java | 18 +-
hadoop/src/test/resources/logback.xml | 2 +-
hive-connector/pom.xml | 12 +-
.../apache/iotdb/hive/TSFHiveInputFormatTest.java | 9 +-
.../apache/iotdb/hive/TSFHiveRecordReaderTest.java | 3 +-
.../apache/iotdb/hive/constant/TestConstant.java | 18 +-
jdbc/README.md | 3 +-
jdbc/pom.xml | 2 +-
.../main/java/org/apache/iotdb/jdbc/Constant.java | 3 -
.../org/apache/iotdb/jdbc/IoTDBConnection.java | 150 +-
.../apache/iotdb/jdbc/IoTDBDatabaseMetadata.java | 44 +-
.../iotdb/jdbc/IoTDBMetadataResultMetadata.java | 2 +-
.../apache/iotdb/jdbc/IoTDBMetadataResultSet.java | 5 -
.../jdbc/IoTDBPreparedInsertionStatement.java | 15 +-
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 65 +-
.../org/apache/iotdb/jdbc/IoTDBQueryResultSet.java | 321 ++--
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 125 +-
.../src/main/java/org/apache/iotdb/jdbc/Utils.java | 170 +-
.../test/java/org/apache/iotdb/jdbc/BatchTest.java | 34 +-
.../org/apache/iotdb/jdbc/IoTDBConnectionTest.java | 10 +-
.../iotdb/jdbc/IoTDBDatabaseMetadataTest.java | 4 +-
.../iotdb/jdbc/IoTDBPreparedStatementTest.java | 38 +-
.../apache/iotdb/jdbc/IoTDBQueryResultSetTest.java | 142 +-
.../org/apache/iotdb/jdbc/IoTDBStatementTest.java | 35 +-
.../test/java/org/apache/iotdb/jdbc/UtilsTest.java | 228 +--
pom.xml | 19 +-
server/pom.xml | 15 +-
.../resources/conf/iotdb-engine.properties | 2 +-
.../org/apache/iotdb/db/sql/parse/TqlLexer.g | 577 ------
.../org/apache/iotdb/db/sql/parse/TqlParser.g | 824 --------
.../org/apache/iotdb/db/qp/strategy/SqlBase.g4 | 862 +++++++++
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 27 +-
.../org/apache/iotdb/db/conf/IoTDBConstant.java | 19 +-
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 4 +-
.../iotdb/db/conf/adapter/CompressionRatio.java | 12 +-
.../org/apache/iotdb/db/engine/StorageEngine.java | 30 +-
.../engine/storagegroup/StorageGroupProcessor.java | 300 ++-
.../db/engine/storagegroup/TsFileResource.java | 8 +
.../exception/query/IllegalASTFormatException.java | 68 -
.../runtime/SQLParserException.java} | 30 +-
.../org/apache/iotdb/db/metadata/MManager.java | 23 +-
.../org/apache/iotdb/db/qp/QueryProcessor.java | 52 +-
.../apache/iotdb/db/qp/constant/SQLConstant.java | 29 +-
.../iotdb/db/qp/constant/TqlParserConstant.java | 71 -
.../qp/executor/AbstractQueryProcessExecutor.java | 113 +-
.../db/qp/executor/IQueryProcessExecutor.java | 3 +-
.../iotdb/db/qp/executor/QueryProcessExecutor.java | 263 ++-
.../org/apache/iotdb/db/qp/logical/Operator.java | 2 +-
.../db/qp/logical/crud/BasicFunctionOperator.java | 5 +-
.../db/qp/logical/crud/BasicOperatorType.java | 5 +-
.../iotdb/db/qp/logical/crud/QueryOperator.java | 70 +-
.../iotdb/db/qp/logical/sys/LoadFilesOperator.java | 67 +
.../sys/MoveFileOperator.java} | 33 +-
.../logical/sys/RemoveFileOperator.java} | 37 +-
.../iotdb/db/qp/physical/crud/GroupByPlan.java | 41 +-
.../iotdb/db/qp/physical/crud/QueryPlan.java | 23 +
.../db/qp/physical/sys/LoadConfigurationPlan.java | 3 +-
.../sys/{ShowPlan.java => OperateFilePlan.java} | 56 +-
.../apache/iotdb/db/qp/physical/sys/ShowPlan.java | 2 +-
.../iotdb/db/qp/strategy/LogicalGenerator.java | 1952 +++++++++----------
.../strategy/LogicalGeneratorError.java} | 29 +-
.../apache/iotdb/db/qp/strategy/ParseDriver.java | 76 +
.../iotdb/db/qp/strategy/PhysicalGenerator.java | 41 +-
.../qp/strategy/optimizer/ConcatPathOptimizer.java | 3 +-
.../iotdb/db/query/context/QueryContext.java | 10 +-
.../{JobFileManager.java => QueryFileManager.java} | 50 +-
.../db/query/control/QueryResourceManager.java | 35 +-
.../db/query/dataset/DeviceIterateDataSet.java | 24 +-
.../dataset/EngineDataSetWithValueFilter.java | 4 +-
.../dataset/EngineDataSetWithoutValueFilter.java | 13 +-
.../apache/iotdb/db/query/dataset/ListDataSet.java | 5 +-
.../{ListDataSet.java => SingleDataSet.java} | 25 +-
.../dataset/groupby/GroupByEngineDataSet.java | 67 +-
.../groupby/GroupByWithValueFilterDataSet.java | 30 +-
.../groupby/GroupByWithoutValueFilterDataSet.java | 19 +-
.../db/query/executor/AggregateEngineExecutor.java | 2 +-
.../iotdb/db/query/executor/EngineExecutor.java | 2 +-
.../iotdb/db/query/executor/EngineQueryRouter.java | 61 +-
.../db/query/executor/FillEngineExecutor.java | 5 +-
.../db/query/executor/IEngineQueryRouter.java | 15 +-
.../db/query/externalsort/ExternalSortJob.java | 12 +-
.../externalsort/ExternalSortJobScheduler.java | 10 +-
.../externalsort/SimpleExternalSortEngine.java | 2 +-
.../java/org/apache/iotdb/db/query/fill/IFill.java | 2 +-
.../resourceRelated/UnseqResourceMergeReader.java | 2 +-
.../UnseqResourceReaderByTimestamp.java | 2 +-
.../SeriesReaderWithoutValueFilter.java | 9 +-
.../java/org/apache/iotdb/db/service/IoTDB.java | 1 -
.../apache/iotdb/db/service/MetricsService.java | 290 +--
.../org/apache/iotdb/db/service/StaticResps.java | 89 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 792 ++++----
.../apache/iotdb/db/sql/parse/AstErrorNode.java | 58 -
.../org/apache/iotdb/db/sql/parse/AstNode.java | 362 ----
.../apache/iotdb/db/sql/parse/AstNodeOrigin.java | 110 --
.../java/org/apache/iotdb/db/sql/parse/Node.java | 42 -
.../org/apache/iotdb/db/sql/parse/ParseDriver.java | 192 --
.../org/apache/iotdb/db/sql/parse/ParseError.java | 52 -
.../org/apache/iotdb/db/sql/parse/ParseUtils.java | 43 -
.../iotdb/db/sync/receiver/load/FileLoader.java | 43 +-
.../{MemEst => memestimation}/MemEstTool.java | 2 +-
.../{MemEst => memestimation}/MemEstToolCmd.java | 2 +-
.../org/apache/iotdb/db/utils/FileLoaderUtils.java | 62 +
.../apache/iotdb/db/utils/QueryDataSetUtils.java | 90 +-
.../writelog/recover/TsFileRecoverPerformer.java | 6 +-
.../iotdb/db/auth/LocalFIleRoleAccessorTest.java | 3 +-
.../iotdb/db/auth/LocalFileRoleManagerTest.java | 3 +-
.../iotdb/db/auth/LocalFileUserAccessorTest.java | 3 +-
.../iotdb/db/auth/LocalFileUserManagerTest.java | 3 +-
.../strategy/DirectoryStrategyTest.java | 4 +-
.../org/apache/iotdb/db/constant/TestConstant.java | 15 +-
.../db/engine/cache/DeviceMetaDataCacheTest.java | 5 +-
.../db/engine/memtable/MemTableFlushTaskTest.java | 5 +-
.../apache/iotdb/db/engine/merge/MergeLogTest.java | 3 +-
.../iotdb/db/engine/merge/MergeOverLapTest.java | 19 +-
.../iotdb/db/engine/merge/MergePerfTest.java | 3 +-
.../iotdb/db/engine/merge/MergeTaskTest.java | 3 +-
.../apache/iotdb/db/engine/merge/MergeTest.java | 13 +-
.../iotdb/db/engine/merge/MergeUpgradeTest.java | 13 +-
.../engine/modification/DeletionFileNodeTest.java | 4 +-
.../engine/modification/ModificationFileTest.java | 6 +-
.../io/LocalTextModificationAccessorTest.java | 6 +-
.../storagegroup/StorageGroupProcessorTest.java | 5 +-
.../iotdb/db/engine/storagegroup/TTLTest.java | 2 +-
.../engine/storagegroup/TsFileProcessorTest.java | 6 +-
.../version/SimpleFileVersionControllerTest.java | 3 +-
.../iotdb/db/integration/IOTDBGroupByIT.java | 269 +--
.../integration/IOTDBGroupByInnerIntervalIT.java | 296 +++
.../apache/iotdb/db/integration/IoTDBCloseIT.java | 12 +-
.../apache/iotdb/db/integration/IoTDBDaemonIT.java | 22 +-
.../iotdb/db/integration/IoTDBGroupbyDeviceIT.java | 28 +-
.../integration/IoTDBLoadExternalTsfileTest.java | 500 +++++
.../iotdb/db/integration/IoTDBMetadataFetchIT.java | 4 +-
.../db/integration/IoTDBMultiStatementsIT.java | 12 +-
.../iotdb/db/integration/IoTDBPreparedStmtIT.java | 2 -
.../iotdb/db/integration/IoTDBQueryDemoIT.java | 94 +-
.../db/integration/IoTDBSequenceDataQueryIT.java | 12 +-
.../iotdb/db/integration/IoTDBSeriesReaderIT.java | 16 +-
.../iotdb/db/metadata/MManagerImproveTest.java | 2 +-
.../org/apache/iotdb/db/qp/QueryProcessorTest.java | 16 +-
.../iotdb/db/qp/plan/LogicalPlanSmallTest.java | 237 ++-
.../apache/iotdb/db/qp/plan/PhysicalPlanTest.java | 95 +-
.../org/apache/iotdb/db/qp/plan/QPUpdateTest.java | 7 +-
.../iotdb/db/qp/plan/TestConcatOptimizer.java | 5 +-
.../iotdb/db/qp/strategy/LogicalGeneratorTest.java | 5 +-
.../apache/iotdb/db/qp/utils/MemIntQpExecutor.java | 19 +-
.../db/query/control/FileReaderManagerTest.java | 9 +-
.../query/executor/GroupByEngineDataSetTest.java | 117 +-
.../query/externalsort/ExternalSortEngineTest.java | 6 +-
...ExternalSortFileSerializerDeserializerTest.java | 3 +-
.../iotdb/db/query/reader/ReaderTestHelper.java | 3 +-
.../org/apache/iotdb/db/sql/TqlParserTest.java | 2021 --------------------
.../apache/iotdb/db/tools/IoTDBWatermarkTest.java | 5 +-
.../org/apache/iotdb/db/tools/WalCheckerTest.java | 9 +-
.../apache/iotdb/db/utils/EnvironmentUtils.java | 15 +-
.../apache/iotdb/db/utils/OpenFileNumUtilTest.java | 3 +-
server/src/test/resources/iotdb-engine.properties | 21 +
server/src/test/resources/logback.xml | 2 +-
service-rpc/pom.xml | 50 +-
service-rpc/rpc-changelist.md | 26 +
.../java/org/apache/iotdb/rpc/TSStatusCode.java | 1 -
service-rpc/src/main/thrift/rpc.thrift | 185 +-
service-rpc/src/pypi/README.md | 63 +
service-rpc/src/pypi/setup.py | 55 +
session/pom.xml | 2 +-
.../java/org/apache/iotdb/session/Session.java | 113 +-
.../org/apache/iotdb/session/SessionDataSet.java | 159 +-
.../org/apache/iotdb/session/SessionUtils.java | 78 +-
.../org/apache/iotdb/session/IoTDBSessionIT.java | 84 +-
.../iotdb/session/utils/EnvironmentUtils.java | 20 +-
session/src/test/resources/iotdb-engine.properties | 20 +
spark-iotdb-connector/Readme.md | 8 +-
spark-iotdb-connector/pom.xml | 2 +-
.../src/test/resources/iotdb-engine.properties | 21 +
.../apache/iotdb/spark/db/EnvironmentUtils.java | 13 +-
.../org/apache/iotdb/spark/db/IoTDBTest.scala | 2 +
spark-tsfile/README.md | 6 +-
spark-tsfile/pom.xml | 4 +-
.../apache/iotdb/spark/constant/TestConstant.java | 17 +-
.../apache/iotdb/spark/tsfile/ConverterTest.scala | 3 +-
.../apache/iotdb/spark/tsfile/HDFSInputTest.java | 4 +-
.../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 3 +-
tsfile/README.md | 2 +-
tsfile/pom.xml | 2 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 6 +-
.../iotdb/tsfile/read/filter/GroupByFilter.java | 101 +
.../tsfile/read/filter/factory/FilterType.java | 2 +-
.../query/dataset/DataSetWithTimeGenerator.java | 5 +-
.../query/dataset/DataSetWithoutTimeGenerator.java | 4 +-
.../tsfile/read/query/dataset/QueryDataSet.java | 59 +-
...meseriesTestConstant.java => TestConstant.java} | 14 +-
.../iotdb/tsfile/file/header/PageHeaderTest.java | 6 +-
.../file/metadata/ChunkGroupMetaDataTest.java | 3 +-
.../file/metadata/TimeSeriesMetadataTest.java | 4 +-
.../file/metadata/TsDeviceMetadataIndexTest.java | 4 +-
.../tsfile/file/metadata/TsFileMetaDataTest.java | 202 +-
.../org/apache/iotdb/tsfile/read/ReadTest.java | 750 ++++----
.../iotdb/tsfile/read/TimePlainEncodeReadTest.java | 733 +++----
.../tsfile/read/filter/GroupByFilterTest.java | 126 ++
.../TsFileGeneratorForSeriesReaderByTimestamp.java | 451 ++---
.../apache/iotdb/tsfile/utils/BytesUtilsTest.java | 22 +-
.../apache/iotdb/tsfile/utils/FileGenerator.java | 429 ++---
.../apache/iotdb/tsfile/utils/FileUtilsTest.java | 10 +-
.../iotdb/tsfile/utils/TsFileGeneratorForTest.java | 7 +-
.../org/apache/iotdb/tsfile/write/PerfTest.java | 7 +-
.../iotdb/tsfile/write/ReadPageInMemTest.java | 3 +-
.../iotdb/tsfile/write/TsFileIOWriterTest.java | 3 +-
.../iotdb/tsfile/write/TsFileReadWriteTest.java | 3 +-
.../org/apache/iotdb/tsfile/write/WriteTest.java | 7 +-
.../iotdb/tsfile/write/writer/PageWriterTest.java | 4 +-
.../write/writer/RestorableTsFileIOWriterTest.java | 4 +-
tsfile/src/test/resources/logback.xml | 2 +-
272 files changed, 8641 insertions(+), 9986 deletions(-)
diff --cc server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index 8feb826,f71b8f3..eaee4bb
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@@ -68,9 -69,8 +70,9 @@@ public class EnvironmentUtils
private static long oldGroupSizeInByte = config.getMemtableSizeThreshold();
public static void cleanEnv() throws IOException, StorageEngineException {
+ NVMSpaceManager.close();
- QueryResourceManager.getInstance().endQueryForGivenJob(TEST_QUERY_JOB_ID);
+ QueryResourceManager.getInstance().endQuery(TEST_QUERY_JOB_ID);
// clear opened file streams
FileReaderManager.getInstance().closeAndRemoveAllOpenedReaders();
@@@ -145,8 -141,8 +145,8 @@@
/**
* disable memory control</br> this function should be called before all code in the setup
*/
- public static void envSetUp() throws StartupException, IOException {
+ public static void envSetUp() throws StartupException {
- IoTDBDescriptor.getInstance().getConfig().setEnableParameterAdapter(false);
+ config.setEnableParameterAdapter(false);
MManager.getInstance().init();
IoTDBConfigDynamicAdapter.getInstance().setInitialized(true);
@@@ -168,10 -164,8 +168,10 @@@
MultiFileLogNodeManager.getInstance().start();
FlushManager.getInstance().start();
MergeManager.getINSTANCE().start();
- TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignJobId();
+ TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId();
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
+
+ NVMSpaceManager.init(config.getNvmDir());
}
private static void createAllDir() {
@@@ -187,10 -181,9 +187,11 @@@
createDir(config.getSystemDir());
// create wal
createDir(config.getWalFolder());
+ // create index
+ createDir(config.getIndexFileDir());
// create query
createDir(config.getQueryDir());
+ createDir(TestConstant.OUTPUT_DATA_DIR);
// create data
for (String dataDir : config.getDataDirs()) {
createDir(dataDir);
[incubator-iotdb] 02/02: update
Posted by ea...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
east pushed a commit to branch nvmlogging
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit c4fd53cbac784838e2d792fc467d1eb77f2ef4e8
Author: mdf369 <95...@qq.com>
AuthorDate: Mon Dec 16 21:34:16 2019 +0800
update
---
.../db/nvm/datastructure/NVMBooleanTVList.java | 197 +++++++++++++++++++++
.../db/nvm/datastructure/NVMDoubleTVList.java | 197 +++++++++++++++++++++
.../iotdb/db/nvm/datastructure/NVMFloatTVList.java | 197 +++++++++++++++++++++
.../iotdb/db/nvm/datastructure/NVMTVList.java | 6 +-
.../iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java | 2 +-
.../apache/iotdb/db/nvm/space/NVMSpaceManager.java | 42 +++--
.../apache/iotdb/db/rescon/TVListAllocator.java | 21 ++-
.../java/org/apache/iotdb/db/service/IoTDB.java | 3 +
.../apache/iotdb/db/utils/EnvironmentUtils.java | 6 +-
9 files changed, 640 insertions(+), 31 deletions(-)
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMBooleanTVList.java b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMBooleanTVList.java
new file mode 100644
index 0000000..96f48d6
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMBooleanTVList.java
@@ -0,0 +1,197 @@
+package org.apache.iotdb.db.nvm.datastructure;
+
+import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool;
+import org.apache.iotdb.db.nvm.space.NVMSpaceManager.NVMSpace;
+import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+public class NVMBooleanTVList extends NVMTVList {
+
+ private List<NVMSpace> values;
+
+ // TODO
+ private boolean[][] sortedValues;
+
+ private boolean pivotValue;
+
+ NVMBooleanTVList() {
+ super();
+ values = new ArrayList<>();
+ }
+
+ @Override
+ public void putBoolean(long timestamp, boolean value) {
+ checkExpansion();
+ int arrayIndex = size / ARRAY_SIZE;
+ int elementIndex = size % ARRAY_SIZE;
+ minTime = minTime <= timestamp ? minTime : timestamp;
+ timestamps.get(arrayIndex).set(elementIndex, timestamp);
+ values.get(arrayIndex).set(elementIndex, value);
+ size++;
+ if (sorted && size > 1 && timestamp < getTime(size - 2)) {
+ sorted = false;
+ }
+ }
+
+ @Override
+ public boolean getBoolean(int index) {
+ if (index >= size) {
+ throw new ArrayIndexOutOfBoundsException(index);
+ }
+ int arrayIndex = index / ARRAY_SIZE;
+ int elementIndex = index % ARRAY_SIZE;
+ return (boolean) values.get(arrayIndex).get(elementIndex);
+ }
+
+ protected void set(int index, long timestamp, boolean value) {
+ if (index >= size) {
+ throw new ArrayIndexOutOfBoundsException(index);
+ }
+ int arrayIndex = index / ARRAY_SIZE;
+ int elementIndex = index % ARRAY_SIZE;
+ timestamps.get(arrayIndex).set(elementIndex, timestamp);
+ values.get(arrayIndex).set(elementIndex, value);
+ }
+
+ @Override
+ public NVMBooleanTVList clone() {
+ NVMBooleanTVList cloneList = new NVMBooleanTVList();
+ cloneAs(cloneList);
+ for (NVMSpace valueSpace : values) {
+ cloneList.values.add(cloneValue(valueSpace));
+ }
+ return cloneList;
+ }
+
+ private NVMSpace cloneValue(NVMSpace valueSpace) {
+ return valueSpace.clone();
+ }
+
+ @Override
+ public void sort() {
+ if (sortedTimestamps == null || sortedTimestamps.length < size) {
+ sortedTimestamps = (long[][]) PrimitiveArrayPool
+ .getInstance().getDataListsByType(TSDataType.INT64, size);
+ }
+ if (sortedValues == null || sortedValues.length < size) {
+ sortedValues = (boolean[][]) PrimitiveArrayPool
+ .getInstance().getDataListsByType(TSDataType.BOOLEAN, size);
+ }
+ sort(0, size);
+ clearSortedValue();
+ clearSortedTime();
+ sorted = true;
+ }
+
+ @Override
+ protected void clearValue() {
+ if (values != null) {
+ for (NVMSpace valueSpace : values) {
+ PrimitiveArrayPool.getInstance().release(valueSpace);
+ }
+ values.clear();
+ }
+ }
+
+ @Override
+ protected void clearSortedValue() {
+ if (sortedValues != null) {
+ for (boolean[] dataArray : sortedValues) {
+ PrimitiveArrayPool.getInstance().release(dataArray);
+ }
+ sortedValues = null;
+ }
+ }
+
+ @Override
+ protected void setFromSorted(int src, int dest) {
+ set(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ }
+
+ @Override
+ protected void set(int src, int dest) {
+ long srcT = getTime(src);
+ boolean srcV = getBoolean(src);
+ set(dest, srcT, srcV);
+ }
+
+ @Override
+ protected void setToSorted(int src, int dest) {
+ sortedTimestamps[dest/ARRAY_SIZE][dest% ARRAY_SIZE] = getTime(src);
+ sortedValues[dest/ARRAY_SIZE][dest%ARRAY_SIZE] = getBoolean(src);
+ }
+
+ @Override
+ protected void reverseRange(int lo, int hi) {
+ hi--;
+ while (lo < hi) {
+ long loT = getTime(lo);
+ boolean loV = getBoolean(lo);
+ long hiT = getTime(hi);
+ boolean hiV = getBoolean(hi);
+ set(lo++, hiT, hiV);
+ set(hi--, loT, loV);
+ }
+ }
+
+ @Override
+ protected void expandValues() {
+ values.add(NVMPrimitiveArrayPool
+ .getInstance().getPrimitiveDataListByType(TSDataType.BOOLEAN));
+ }
+
+ @Override
+ protected void saveAsPivot(int pos) {
+ pivotTime = getTime(pos);
+ pivotValue = getBoolean(pos);
+ }
+
+ @Override
+ protected void setPivotTo(int pos) {
+ set(pos, pivotTime, pivotValue);
+ }
+
+ @Override
+ protected void releaseLastValueArray() {
+ PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1));
+ }
+
+ @Override
+ public void putBooleans(long[] time, boolean[] value) {
+ checkExpansion();
+ int idx = 0;
+ int length = time.length;
+
+ for (int i = 0; i < length; i++) {
+ putBoolean(time[i], value[i]);
+ }
+
+// updateMinTimeAndSorted(time);
+//
+// while (idx < length) {
+// int inputRemaining = length - idx;
+// int arrayIdx = size / ARRAY_SIZE;
+// int elementIdx = size % ARRAY_SIZE;
+// int internalRemaining = ARRAY_SIZE - elementIdx;
+// if (internalRemaining >= inputRemaining) {
+// // the remaining inputs can fit the last array, copy all remaining inputs into last array
+// System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining);
+// System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining);
+// size += inputRemaining;
+// break;
+// } else {
+// // the remaining inputs cannot fit the last array, fill the last array and create a new
+// // one and enter the next loop
+// System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining);
+// System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining);
+// idx += internalRemaining;
+// size += internalRemaining;
+// checkExpansion();
+// }
+// }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMDoubleTVList.java b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMDoubleTVList.java
new file mode 100644
index 0000000..12ffc2e
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMDoubleTVList.java
@@ -0,0 +1,197 @@
+package org.apache.iotdb.db.nvm.datastructure;
+
+import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool;
+import org.apache.iotdb.db.nvm.space.NVMSpaceManager.NVMSpace;
+import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+public class NVMDoubleTVList extends NVMTVList {
+
+ private List<NVMSpace> values;
+
+ // TODO
+ private double[][] sortedValues;
+
+ private double pivotValue;
+
+ NVMDoubleTVList() {
+ super();
+ values = new ArrayList<>();
+ }
+
+ @Override
+ public void putDouble(long timestamp, double value) {
+ checkExpansion();
+ int arrayIndex = size / ARRAY_SIZE;
+ int elementIndex = size % ARRAY_SIZE;
+ minTime = minTime <= timestamp ? minTime : timestamp;
+ timestamps.get(arrayIndex).set(elementIndex, timestamp);
+ values.get(arrayIndex).set(elementIndex, value);
+ size++;
+ if (sorted && size > 1 && timestamp < getTime(size - 2)) {
+ sorted = false;
+ }
+ }
+
+ @Override
+ public double getDouble(int index) {
+ if (index >= size) {
+ throw new ArrayIndexOutOfBoundsException(index);
+ }
+ int arrayIndex = index / ARRAY_SIZE;
+ int elementIndex = index % ARRAY_SIZE;
+ return (double) values.get(arrayIndex).get(elementIndex);
+ }
+
+ protected void set(int index, long timestamp, double value) {
+ if (index >= size) {
+ throw new ArrayIndexOutOfBoundsException(index);
+ }
+ int arrayIndex = index / ARRAY_SIZE;
+ int elementIndex = index % ARRAY_SIZE;
+ timestamps.get(arrayIndex).set(elementIndex, timestamp);
+ values.get(arrayIndex).set(elementIndex, value);
+ }
+
+ @Override
+ public NVMDoubleTVList clone() {
+ NVMDoubleTVList cloneList = new NVMDoubleTVList();
+ cloneAs(cloneList);
+ for (NVMSpace valueSpace : values) {
+ cloneList.values.add(cloneValue(valueSpace));
+ }
+ return cloneList;
+ }
+
+ private NVMSpace cloneValue(NVMSpace valueSpace) {
+ return valueSpace.clone();
+ }
+
+ @Override
+ public void sort() {
+ if (sortedTimestamps == null || sortedTimestamps.length < size) {
+ sortedTimestamps = (long[][]) PrimitiveArrayPool
+ .getInstance().getDataListsByType(TSDataType.INT64, size);
+ }
+ if (sortedValues == null || sortedValues.length < size) {
+ sortedValues = (double[][]) PrimitiveArrayPool
+ .getInstance().getDataListsByType(TSDataType.DOUBLE, size);
+ }
+ sort(0, size);
+ clearSortedValue();
+ clearSortedTime();
+ sorted = true;
+ }
+
+ @Override
+ protected void clearValue() {
+ if (values != null) {
+ for (NVMSpace valueSpace : values) {
+ PrimitiveArrayPool.getInstance().release(valueSpace);
+ }
+ values.clear();
+ }
+ }
+
+ @Override
+ protected void clearSortedValue() {
+ if (sortedValues != null) {
+ for (double[] dataArray : sortedValues) {
+ PrimitiveArrayPool.getInstance().release(dataArray);
+ }
+ sortedValues = null;
+ }
+ }
+
+ @Override
+ protected void setFromSorted(int src, int dest) {
+ set(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ }
+
+ @Override
+ protected void set(int src, int dest) {
+ long srcT = getTime(src);
+ double srcV = getDouble(src);
+ set(dest, srcT, srcV);
+ }
+
+ @Override
+ protected void setToSorted(int src, int dest) {
+ sortedTimestamps[dest/ARRAY_SIZE][dest% ARRAY_SIZE] = getTime(src);
+ sortedValues[dest/ARRAY_SIZE][dest%ARRAY_SIZE] = getDouble(src);
+ }
+
+ @Override
+ protected void reverseRange(int lo, int hi) {
+ hi--;
+ while (lo < hi) {
+ long loT = getTime(lo);
+ double loV = getDouble(lo);
+ long hiT = getTime(hi);
+ double hiV = getDouble(hi);
+ set(lo++, hiT, hiV);
+ set(hi--, loT, loV);
+ }
+ }
+
+ @Override
+ protected void expandValues() {
+ values.add(NVMPrimitiveArrayPool
+ .getInstance().getPrimitiveDataListByType(TSDataType.DOUBLE));
+ }
+
+ @Override
+ protected void saveAsPivot(int pos) {
+ pivotTime = getTime(pos);
+ pivotValue = getDouble(pos);
+ }
+
+ @Override
+ protected void setPivotTo(int pos) {
+ set(pos, pivotTime, pivotValue);
+ }
+
+ @Override
+ protected void releaseLastValueArray() {
+ PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1));
+ }
+
+ @Override
+ public void putDoubles(long[] time, double[] value) {
+ checkExpansion();
+ int idx = 0;
+ int length = time.length;
+
+ for (int i = 0; i < length; i++) {
+ putDouble(time[i], value[i]);
+ }
+
+// updateMinTimeAndSorted(time);
+//
+// while (idx < length) {
+// int inputRemaining = length - idx;
+// int arrayIdx = size / ARRAY_SIZE;
+// int elementIdx = size % ARRAY_SIZE;
+// int internalRemaining = ARRAY_SIZE - elementIdx;
+// if (internalRemaining >= inputRemaining) {
+// // the remaining inputs can fit the last array, copy all remaining inputs into last array
+// System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining);
+// System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining);
+// size += inputRemaining;
+// break;
+// } else {
+// // the remaining inputs cannot fit the last array, fill the last array and create a new
+// // one and enter the next loop
+// System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining);
+// System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining);
+// idx += internalRemaining;
+// size += internalRemaining;
+// checkExpansion();
+// }
+// }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMFloatTVList.java b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMFloatTVList.java
new file mode 100644
index 0000000..9e527fc
--- /dev/null
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMFloatTVList.java
@@ -0,0 +1,197 @@
+package org.apache.iotdb.db.nvm.datastructure;
+
+import static org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool.ARRAY_SIZE;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.iotdb.db.nvm.rescon.NVMPrimitiveArrayPool;
+import org.apache.iotdb.db.nvm.space.NVMSpaceManager.NVMSpace;
+import org.apache.iotdb.db.rescon.PrimitiveArrayPool;
+import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+
+public class NVMFloatTVList extends NVMTVList {
+
+ private List<NVMSpace> values;
+
+ // TODO
+ private float[][] sortedValues;
+
+ private float pivotValue;
+
+ NVMFloatTVList() {
+ super();
+ values = new ArrayList<>();
+ }
+
+ @Override
+ public void putFloat(long timestamp, float value) {
+ checkExpansion();
+ int arrayIndex = size / ARRAY_SIZE;
+ int elementIndex = size % ARRAY_SIZE;
+ minTime = minTime <= timestamp ? minTime : timestamp;
+ timestamps.get(arrayIndex).set(elementIndex, timestamp);
+ values.get(arrayIndex).set(elementIndex, value);
+ size++;
+ if (sorted && size > 1 && timestamp < getTime(size - 2)) {
+ sorted = false;
+ }
+ }
+
+ @Override
+ public float getFloat(int index) {
+ if (index >= size) {
+ throw new ArrayIndexOutOfBoundsException(index);
+ }
+ int arrayIndex = index / ARRAY_SIZE;
+ int elementIndex = index % ARRAY_SIZE;
+ return (float) values.get(arrayIndex).get(elementIndex);
+ }
+
+ protected void set(int index, long timestamp, float value) {
+ if (index >= size) {
+ throw new ArrayIndexOutOfBoundsException(index);
+ }
+ int arrayIndex = index / ARRAY_SIZE;
+ int elementIndex = index % ARRAY_SIZE;
+ timestamps.get(arrayIndex).set(elementIndex, timestamp);
+ values.get(arrayIndex).set(elementIndex, value);
+ }
+
+ @Override
+ public NVMFloatTVList clone() {
+ NVMFloatTVList cloneList = new NVMFloatTVList();
+ cloneAs(cloneList);
+ for (NVMSpace valueSpace : values) {
+ cloneList.values.add(cloneValue(valueSpace));
+ }
+ return cloneList;
+ }
+
+ private NVMSpace cloneValue(NVMSpace valueSpace) {
+ return valueSpace.clone();
+ }
+
+ @Override
+ public void sort() {
+ if (sortedTimestamps == null || sortedTimestamps.length < size) {
+ sortedTimestamps = (long[][]) PrimitiveArrayPool
+ .getInstance().getDataListsByType(TSDataType.INT64, size);
+ }
+ if (sortedValues == null || sortedValues.length < size) {
+ sortedValues = (float[][]) PrimitiveArrayPool
+ .getInstance().getDataListsByType(TSDataType.FLOAT, size);
+ }
+ sort(0, size);
+ clearSortedValue();
+ clearSortedTime();
+ sorted = true;
+ }
+
+ @Override
+ protected void clearValue() {
+ if (values != null) {
+ for (NVMSpace valueSpace : values) {
+ PrimitiveArrayPool.getInstance().release(valueSpace);
+ }
+ values.clear();
+ }
+ }
+
+ @Override
+ protected void clearSortedValue() {
+ if (sortedValues != null) {
+ for (float[] dataArray : sortedValues) {
+ PrimitiveArrayPool.getInstance().release(dataArray);
+ }
+ sortedValues = null;
+ }
+ }
+
+ @Override
+ protected void setFromSorted(int src, int dest) {
+ set(dest, sortedTimestamps[src/ARRAY_SIZE][src%ARRAY_SIZE], sortedValues[src/ARRAY_SIZE][src%ARRAY_SIZE]);
+ }
+
+ @Override
+ protected void set(int src, int dest) {
+ long srcT = getTime(src);
+ float srcV = getFloat(src);
+ set(dest, srcT, srcV);
+ }
+
+ @Override
+ protected void setToSorted(int src, int dest) {
+ sortedTimestamps[dest/ARRAY_SIZE][dest% ARRAY_SIZE] = getTime(src);
+ sortedValues[dest/ARRAY_SIZE][dest%ARRAY_SIZE] = getFloat(src);
+ }
+
+ @Override
+ protected void reverseRange(int lo, int hi) {
+ hi--;
+ while (lo < hi) {
+ long loT = getTime(lo);
+ float loV = getFloat(lo);
+ long hiT = getTime(hi);
+ float hiV = getFloat(hi);
+ set(lo++, hiT, hiV);
+ set(hi--, loT, loV);
+ }
+ }
+
+ @Override
+ protected void expandValues() {
+ values.add(NVMPrimitiveArrayPool
+ .getInstance().getPrimitiveDataListByType(TSDataType.FLOAT));
+ }
+
+ @Override
+ protected void saveAsPivot(int pos) {
+ pivotTime = getTime(pos);
+ pivotValue = getFloat(pos);
+ }
+
+ @Override
+ protected void setPivotTo(int pos) {
+ set(pos, pivotTime, pivotValue);
+ }
+
+ @Override
+ protected void releaseLastValueArray() {
+ PrimitiveArrayPool.getInstance().release(values.remove(values.size() - 1));
+ }
+
+ @Override
+ public void putFloats(long[] time, float[] value) {
+ checkExpansion();
+ int idx = 0;
+ int length = time.length;
+
+ for (int i = 0; i < length; i++) {
+ putFloat(time[i], value[i]);
+ }
+
+// updateMinTimeAndSorted(time);
+//
+// while (idx < length) {
+// int inputRemaining = length - idx;
+// int arrayIdx = size / ARRAY_SIZE;
+// int elementIdx = size % ARRAY_SIZE;
+// int internalRemaining = ARRAY_SIZE - elementIdx;
+// if (internalRemaining >= inputRemaining) {
+// // the remaining inputs can fit the last array, copy all remaining inputs into last array
+// System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, inputRemaining);
+// System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, inputRemaining);
+// size += inputRemaining;
+// break;
+// } else {
+// // the remaining inputs cannot fit the last array, fill the last array and create a new
+// // one and enter the next loop
+// System.arraycopy(time, idx, timestamps.get(arrayIdx), elementIdx, internalRemaining);
+// System.arraycopy(value, idx, values.get(arrayIdx), elementIdx, internalRemaining);
+// idx += internalRemaining;
+// size += internalRemaining;
+// checkExpansion();
+// }
+// }
+ }
+}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMTVList.java b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMTVList.java
index 45df453..e7ffebd 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMTVList.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/datastructure/NVMTVList.java
@@ -108,15 +108,15 @@ public abstract class NVMTVList extends AbstractTVList {
// TODO
// return new BinaryTVList();
case FLOAT:
-// return new FloatTVList();
+ return new NVMFloatTVList();
case INT32:
return new NVMIntTVList();
case INT64:
return new NVMLongTVList();
case DOUBLE:
-// return new DoubleTVList();
+ return new NVMDoubleTVList();
case BOOLEAN:
-// return new BooleanTVList();
+ return new NVMBooleanTVList();
}
return null;
}
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
index 0647217..7cfef5a 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/rescon/NVMPrimitiveArrayPool.java
@@ -40,7 +40,7 @@ public class NVMPrimitiveArrayPool {
long size = NVMSpaceManager.getPrimitiveTypeByteSize(dataType);
if (nvmSpace == null) {
- nvmSpace = NVMSpaceManager.allocate(size * ARRAY_SIZE, dataType);
+ nvmSpace = NVMSpaceManager.getInstance().allocate(size * ARRAY_SIZE, dataType);
}
return nvmSpace;
diff --git a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java
index 24d3d0d..d504e9f 100644
--- a/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java
+++ b/server/src/main/java/org/apache/iotdb/db/nvm/space/NVMSpaceManager.java
@@ -6,8 +6,11 @@ import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.util.concurrent.atomic.AtomicLong;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
+import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
+import org.apache.iotdb.tsfile.fileSystem.FSFactoryProducer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -15,24 +18,31 @@ public class NVMSpaceManager {
private static final Logger logger = LoggerFactory.getLogger(NVMSpaceManager.class);
- private static String NVM_PATH;
- private static FileChannel nvmFileChannel;
- private static final MapMode MAP_MODE = MapMode.READ_WRITE;
- private static long nvmSize;
- private static AtomicLong curOffset = new AtomicLong(0L);
+ private final static NVMSpaceManager INSTANCE = new NVMSpaceManager();
- public static void init(String dir) {
+ private String NVM_PATH;
+ private FileChannel nvmFileChannel;
+ private final MapMode MAP_MODE = MapMode.READ_WRITE;
+ private long nvmSize;
+ private AtomicLong curOffset = new AtomicLong(0L);
+
+ private NVMSpaceManager() {
+// init();
+ }
+
+ public void init() throws StartupException {
try {
- NVM_PATH = dir + "/nvmFile";
+ NVM_PATH = IoTDBDescriptor.getInstance().getConfig().getNvmDir() + "/nvmFile";
+ FSFactoryProducer.getFSFactory().getFile(IoTDBDescriptor.getInstance().getConfig().getNvmDir()).mkdirs();
nvmFileChannel = new RandomAccessFile(NVM_PATH, "rw").getChannel();
nvmSize = nvmFileChannel.size();
} catch (IOException e) {
logger.error("Fail to open NVM space at {}.", NVM_PATH, e);
- // TODO deal with error
+ throw new StartupException(e);
}
}
- public static void close() throws IOException {
+ public void close() throws IOException {
nvmFileChannel.close();
}
@@ -63,7 +73,7 @@ public class NVMSpaceManager {
return size >> 3;
}
- public static NVMSpace allocate(long size, TSDataType dataType) {
+ public NVMSpace allocate(long size, TSDataType dataType) {
long offset = curOffset.getAndAdd(size);
if (offset + size > nvmSize) {
// TODO throw exception
@@ -78,7 +88,7 @@ public class NVMSpaceManager {
}
}
- public static class NVMSpace {
+ public class NVMSpace {
private long offset;
private long size;
@@ -106,7 +116,7 @@ public class NVMSpaceManager {
@Override
public NVMSpace clone() {
- NVMSpace cloneSpace = NVMSpaceManager.allocate(size, dataType);
+ NVMSpace cloneSpace = NVMSpaceManager.getInstance().allocate(size, dataType);
int position = byteBuffer.position();
byteBuffer.rewind();
cloneSpace.getByteBuffer().put(byteBuffer);
@@ -168,11 +178,7 @@ public class NVMSpaceManager {
}
}
- public static String getNvmPath() {
- return NVM_PATH;
- }
-
- public static void setNvmPath(String nvmPath) {
- NVM_PATH = nvmPath;
+ public static NVMSpaceManager getInstance() {
+ return INSTANCE;
}
}
diff --git a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
index f32950a..f7f8d91 100644
--- a/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
+++ b/server/src/main/java/org/apache/iotdb/db/rescon/TVListAllocator.java
@@ -26,6 +26,9 @@ import java.util.Queue;
import org.apache.iotdb.db.conf.IoTDBConstant;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.nvm.datastructure.AbstractTVList;
+import org.apache.iotdb.db.nvm.datastructure.NVMBooleanTVList;
+import org.apache.iotdb.db.nvm.datastructure.NVMDoubleTVList;
+import org.apache.iotdb.db.nvm.datastructure.NVMFloatTVList;
import org.apache.iotdb.db.nvm.datastructure.NVMIntTVList;
import org.apache.iotdb.db.nvm.datastructure.NVMLongTVList;
import org.apache.iotdb.db.nvm.datastructure.NVMTVList;
@@ -81,10 +84,20 @@ public class TVListAllocator implements TVListAllocatorMBean, IService {
public synchronized void release(AbstractTVList list) {
list.clear();
- if (list instanceof NVMIntTVList) {
- nvmTVListCache.get(TSDataType.INT32).add((NVMTVList) list);
- } else if (list instanceof NVMLongTVList) {
- nvmTVListCache.get(TSDataType.INT64).add((NVMTVList) list);
+ if (list instanceof NVMTVList) {
+ NVMTVList tvList = (NVMTVList) list;
+ if (list instanceof NVMIntTVList) {
+ nvmTVListCache.get(TSDataType.INT32).add(tvList);
+ } else if (list instanceof NVMLongTVList) {
+ nvmTVListCache.get(TSDataType.INT64).add(tvList);
+ } else if (list instanceof NVMFloatTVList) {
+ nvmTVListCache.get(TSDataType.FLOAT).add(tvList);
+ } else if (list instanceof NVMDoubleTVList) {
+ nvmTVListCache.get(TSDataType.DOUBLE).add(tvList);
+ } else if (list instanceof NVMBooleanTVList) {
+ nvmTVListCache.get(TSDataType.BOOLEAN).add(tvList);
+ }
+ // TODO Binary
} else {
TVList tvList = (TVList) list;
diff --git a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
index b191fa8..0095c63 100644
--- a/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
+++ b/server/src/main/java/org/apache/iotdb/db/service/IoTDB.java
@@ -32,6 +32,7 @@ import org.apache.iotdb.db.engine.flush.FlushManager;
import org.apache.iotdb.db.exception.StartupException;
import org.apache.iotdb.db.metadata.MManager;
import org.apache.iotdb.db.monitor.StatMonitor;
+import org.apache.iotdb.db.nvm.space.NVMSpaceManager;
import org.apache.iotdb.db.rescon.TVListAllocator;
import org.apache.iotdb.db.sync.receiver.SyncServerManager;
import org.apache.iotdb.db.writelog.manager.MultiFileLogNodeManager;
@@ -107,6 +108,8 @@ public class IoTDB implements IoTDBMBean {
StatMonitor.getInstance().recovery();
}
+ NVMSpaceManager.getInstance().init();
+
logger.info("IoTDB is set up.");
}
diff --git a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
index eaee4bb..1a8111c 100644
--- a/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
+++ b/server/src/test/java/org/apache/iotdb/db/utils/EnvironmentUtils.java
@@ -70,7 +70,7 @@ public class EnvironmentUtils {
private static long oldGroupSizeInByte = config.getMemtableSizeThreshold();
public static void cleanEnv() throws IOException, StorageEngineException {
- NVMSpaceManager.close();
+ NVMSpaceManager.getInstance().close();
QueryResourceManager.getInstance().endQuery(TEST_QUERY_JOB_ID);
@@ -170,8 +170,6 @@ public class EnvironmentUtils {
MergeManager.getINSTANCE().start();
TEST_QUERY_JOB_ID = QueryResourceManager.getInstance().assignQueryId();
TEST_QUERY_CONTEXT = new QueryContext(TEST_QUERY_JOB_ID);
-
- NVMSpaceManager.init(config.getNvmDir());
}
private static void createAllDir() {
@@ -187,8 +185,6 @@ public class EnvironmentUtils {
createDir(config.getSystemDir());
// create wal
createDir(config.getWalFolder());
- // create index
- createDir(config.getIndexFileDir());
// create query
createDir(config.getQueryDir());
createDir(TestConstant.OUTPUT_DATA_DIR);