You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/05/15 08:51:07 UTC
[incubator-iotdb] branch cluster updated (77a9bd3 -> 6ddf6fd)
This is an automated email from the ASF dual-hosted git repository.
lta pushed a change to branch cluster
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git.
from 77a9bd3 fix sonar issues
add 3568b32 [IOTDB-64] TsFile-Spark-Connector update (#108)
add b1b35f6 fix bug about BOOLEAN
add bf62685 Update QueryConstant.java
add ec72e4e Merge pull request #173 from LeiRui/master
add f8754f9 add the print of information about chunkgroups for debug (#175)
add 451e87d change print to log (#176)
add 0043172 Update README.md
add e7886ca Refactor iotdb code for cluster (#172)
new 6ddf6fd merge master
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../Documentation/UserGuideV0.7.0/7-Tools-spark.md | 286 ++++----
.../java/org/apache/iotdb/db/metadata/MGraph.java | 2 +-
.../org/apache/iotdb/db/metadata/Metadata.java | 102 +--
.../db/qp/executor/IQueryProcessExecutor.java | 10 +-
.../dataset/groupby/GroupByEngineDataSet.java | 5 +-
.../AbstractExecutorWithoutTimeGenerator.java | 4 +
.../EngineExecutorWithoutTimeGenerator.java | 3 +-
.../timegenerator/AbstractNodeConstructor.java | 7 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 1 +
jdbc/README.md | 4 +-
pom.xml | 4 +-
spark/README.md | 407 +++++++----
spark/pom.xml | 7 +-
.../org/apache/iotdb/tsfile/io/CreateTSFile.java | 150 ----
.../java/org/apache/iotdb/tsfile/io/HDFSInput.java | 147 ++++
.../apache/iotdb/tsfile/io/HDFSInputStream.java | 111 ---
.../io/{HDFSOutputStream.java => HDFSOutput.java} | 52 +-
.../apache/iotdb/tsfile/io/TsFileOutputFormat.java | 10 +-
.../apache/iotdb/tsfile/io/TsFileRecordWriter.java | 19 +-
.../java/org/apache/iotdb/tsfile/qp/Executor.java | 51 --
.../org/apache/iotdb/tsfile/qp/QueryProcessor.java | 153 -----
.../iotdb/tsfile/qp/common/BasicOperator.java | 75 --
.../iotdb/tsfile/qp/common/FilterOperator.java | 157 -----
.../apache/iotdb/tsfile/qp/common/Operator.java | 47 --
.../apache/iotdb/tsfile/qp/common/SQLConstant.java | 150 ----
.../apache/iotdb/tsfile/qp/common/SingleQuery.java | 63 --
.../apache/iotdb/tsfile/qp/common/TSQueryPlan.java | 63 --
.../qp/exception/BasicOperatorException.java | 34 -
.../tsfile/qp/exception/DNFOptimizeException.java | 34 -
.../qp/exception/LogicalOptimizeException.java | 33 -
.../tsfile/qp/exception/MergeFilterException.java | 30 -
.../qp/exception/QueryOperatorException.java | 29 -
.../qp/exception/QueryProcessorException.java | 41 --
.../tsfile/qp/exception/RemoveNotException.java | 34 -
.../tsfile/qp/optimizer/DNFFilterOptimizer.java | 157 -----
.../tsfile/qp/optimizer/IFilterOptimizer.java | 34 -
.../qp/optimizer/MergeSingleFilterOptimizer.java | 141 ----
.../tsfile/qp/optimizer/PhysicalOptimizer.java | 228 ------
.../tsfile/qp/optimizer/RemoveNotOptimizer.java | 108 ---
.../scala/org/apache/iotdb/tsfile/Converter.scala | 764 ++++++++++++---------
.../org/apache/iotdb/tsfile/DefaultSource.scala | 178 ++---
.../apache/iotdb/tsfile/TsFileOutputWriter.scala | 52 +-
.../apache/iotdb/tsfile/TsFileWriterFactory.scala | 42 +-
.../scala/org/apache/iotdb/tsfile/package.scala | 36 +-
spark/src/test/resources/test.tsfile | Bin 1406 -> 0 bytes
.../cn/edu/tsinghua/tsfile/ConverterTest.scala | 130 ----
.../scala/cn/edu/tsinghua/tsfile/TSFileSuit.scala | 194 ------
.../scala/org/apache/iotdb/tool/TsFileExample.java | 106 +++
.../scala/org/apache/iotdb/tool/TsFileWrite.java | 215 ++++++
.../org/apache/iotdb/tsfile/ConverterTest.scala | 266 +++++++
.../org/apache/iotdb/tsfile/HDFSInputTest.java | 79 +++
.../scala/org/apache/iotdb/tsfile/TSFileSuit.scala | 217 ++++++
.../tsfile/common/constant/QueryConstant.java | 9 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../apache/iotdb/tsfile/read/common/TimeRange.java | 289 ++++++++
.../tsfile/read/controller/MetadataQuerier.java | 40 ++
.../read/controller/MetadataQuerierByFileImpl.java | 132 +++-
.../tsfile/read/expression/ExpressionType.java | 29 +-
.../query/executor/ExecutorWithTimeGenerator.java | 33 +-
.../tsfile/read/query/executor/TsFileExecutor.java | 64 +-
.../apache/iotdb/tsfile/write/TsFileWriter.java | 12 +
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 11 +
.../iotdb/tsfile/read/ReadInPartitionTest.java | 194 ++++++
.../org/apache/iotdb/tsfile/read/ReadTest.java | 2 +-
.../iotdb/tsfile/read/common/TimeRangeTest.java | 225 ++++++
.../controller/MetadataQuerierByFileImplTest.java | 57 +-
66 files changed, 3163 insertions(+), 3208 deletions(-)
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/io/CreateTSFile.java
create mode 100644 spark/src/main/java/org/apache/iotdb/tsfile/io/HDFSInput.java
delete mode 100644 spark/src/main/java/org/apache/iotdb/tsfile/io/HDFSInputStream.java
rename spark/src/main/java/org/apache/iotdb/tsfile/io/{HDFSOutputStream.java => HDFSOutput.java} (67%)
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/Executor.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/QueryProcessor.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/common/BasicOperator.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/common/FilterOperator.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/common/Operator.java
delete mode 100644 spark/src/main/java/org/apache/iotdb/tsfile/qp/common/SQLConstant.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/common/SingleQuery.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/common/TSQueryPlan.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/exception/BasicOperatorException.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/exception/DNFOptimizeException.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/exception/LogicalOptimizeException.java
delete mode 100644 spark/src/main/java/org/apache/iotdb/tsfile/qp/exception/MergeFilterException.java
delete mode 100644 spark/src/main/java/org/apache/iotdb/tsfile/qp/exception/QueryOperatorException.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/exception/QueryProcessorException.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/exception/RemoveNotException.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/optimizer/DNFFilterOptimizer.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/optimizer/IFilterOptimizer.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/optimizer/MergeSingleFilterOptimizer.java
delete mode 100644 spark/src/main/java/org/apache/iotdb/tsfile/qp/optimizer/PhysicalOptimizer.java
delete mode 100755 spark/src/main/java/org/apache/iotdb/tsfile/qp/optimizer/RemoveNotOptimizer.java
delete mode 100644 spark/src/test/resources/test.tsfile
delete mode 100644 spark/src/test/scala/cn/edu/tsinghua/tsfile/ConverterTest.scala
delete mode 100644 spark/src/test/scala/cn/edu/tsinghua/tsfile/TSFileSuit.scala
create mode 100644 spark/src/test/scala/org/apache/iotdb/tool/TsFileExample.java
create mode 100644 spark/src/test/scala/org/apache/iotdb/tool/TsFileWrite.java
create mode 100644 spark/src/test/scala/org/apache/iotdb/tsfile/ConverterTest.scala
create mode 100644 spark/src/test/scala/org/apache/iotdb/tsfile/HDFSInputTest.java
create mode 100644 spark/src/test/scala/org/apache/iotdb/tsfile/TSFileSuit.scala
create mode 100644 tsfile/src/main/java/org/apache/iotdb/tsfile/read/common/TimeRange.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/read/ReadInPartitionTest.java
create mode 100644 tsfile/src/test/java/org/apache/iotdb/tsfile/read/common/TimeRangeTest.java
[incubator-iotdb] 01/01: merge master
Posted by lt...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch cluster
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit 6ddf6fd10f5886d6ff3b7625348e5cff06a77e5a
Merge: 77a9bd3 e7886ca
Author: lta <li...@163.com>
AuthorDate: Wed May 15 16:50:39 2019 +0800
merge master
.../Documentation/UserGuideV0.7.0/7-Tools-spark.md | 286 ++++----
.../java/org/apache/iotdb/db/metadata/MGraph.java | 2 +-
.../org/apache/iotdb/db/metadata/Metadata.java | 102 +--
.../db/qp/executor/IQueryProcessExecutor.java | 10 +-
.../dataset/groupby/GroupByEngineDataSet.java | 5 +-
.../AbstractExecutorWithoutTimeGenerator.java | 4 +
.../EngineExecutorWithoutTimeGenerator.java | 3 +-
.../timegenerator/AbstractNodeConstructor.java | 7 +
.../org/apache/iotdb/db/service/TSServiceImpl.java | 1 +
jdbc/README.md | 4 +-
pom.xml | 4 +-
spark/README.md | 407 +++++++----
spark/pom.xml | 7 +-
.../org/apache/iotdb/tsfile/io/CreateTSFile.java | 150 ----
.../java/org/apache/iotdb/tsfile/io/HDFSInput.java | 147 ++++
.../apache/iotdb/tsfile/io/HDFSInputStream.java | 111 ---
.../io/{HDFSOutputStream.java => HDFSOutput.java} | 52 +-
.../apache/iotdb/tsfile/io/TsFileOutputFormat.java | 10 +-
.../apache/iotdb/tsfile/io/TsFileRecordWriter.java | 19 +-
.../java/org/apache/iotdb/tsfile/qp/Executor.java | 51 --
.../org/apache/iotdb/tsfile/qp/QueryProcessor.java | 153 -----
.../iotdb/tsfile/qp/common/BasicOperator.java | 75 --
.../iotdb/tsfile/qp/common/FilterOperator.java | 157 -----
.../apache/iotdb/tsfile/qp/common/Operator.java | 47 --
.../apache/iotdb/tsfile/qp/common/SQLConstant.java | 150 ----
.../apache/iotdb/tsfile/qp/common/SingleQuery.java | 63 --
.../apache/iotdb/tsfile/qp/common/TSQueryPlan.java | 63 --
.../qp/exception/BasicOperatorException.java | 34 -
.../tsfile/qp/exception/DNFOptimizeException.java | 34 -
.../qp/exception/LogicalOptimizeException.java | 33 -
.../tsfile/qp/exception/MergeFilterException.java | 30 -
.../qp/exception/QueryOperatorException.java | 29 -
.../qp/exception/QueryProcessorException.java | 41 --
.../tsfile/qp/exception/RemoveNotException.java | 34 -
.../tsfile/qp/optimizer/DNFFilterOptimizer.java | 157 -----
.../tsfile/qp/optimizer/IFilterOptimizer.java | 34 -
.../qp/optimizer/MergeSingleFilterOptimizer.java | 141 ----
.../tsfile/qp/optimizer/PhysicalOptimizer.java | 228 ------
.../tsfile/qp/optimizer/RemoveNotOptimizer.java | 108 ---
.../scala/org/apache/iotdb/tsfile/Converter.scala | 764 ++++++++++++---------
.../org/apache/iotdb/tsfile/DefaultSource.scala | 178 ++---
.../apache/iotdb/tsfile/TsFileOutputWriter.scala | 52 +-
.../apache/iotdb/tsfile/TsFileWriterFactory.scala | 42 +-
.../scala/org/apache/iotdb/tsfile/package.scala | 36 +-
spark/src/test/resources/test.tsfile | Bin 1406 -> 0 bytes
.../cn/edu/tsinghua/tsfile/ConverterTest.scala | 130 ----
.../scala/cn/edu/tsinghua/tsfile/TSFileSuit.scala | 194 ------
.../scala/org/apache/iotdb/tool/TsFileExample.java | 106 +++
.../scala/org/apache/iotdb/tool/TsFileWrite.java | 215 ++++++
.../org/apache/iotdb/tsfile/ConverterTest.scala | 266 +++++++
.../org/apache/iotdb/tsfile/HDFSInputTest.java | 79 +++
.../scala/org/apache/iotdb/tsfile/TSFileSuit.scala | 217 ++++++
.../tsfile/common/constant/QueryConstant.java | 9 +
.../iotdb/tsfile/read/TsFileSequenceReader.java | 2 +-
.../apache/iotdb/tsfile/read/common/TimeRange.java | 289 ++++++++
.../tsfile/read/controller/MetadataQuerier.java | 40 ++
.../read/controller/MetadataQuerierByFileImpl.java | 132 +++-
.../tsfile/read/expression/ExpressionType.java | 29 +-
.../query/executor/ExecutorWithTimeGenerator.java | 33 +-
.../tsfile/read/query/executor/TsFileExecutor.java | 64 +-
.../apache/iotdb/tsfile/write/TsFileWriter.java | 12 +
.../iotdb/tsfile/write/writer/TsFileIOWriter.java | 11 +
.../iotdb/tsfile/read/ReadInPartitionTest.java | 194 ++++++
.../org/apache/iotdb/tsfile/read/ReadTest.java | 2 +-
.../iotdb/tsfile/read/common/TimeRangeTest.java | 225 ++++++
.../controller/MetadataQuerierByFileImplTest.java | 57 +-
66 files changed, 3163 insertions(+), 3208 deletions(-)
diff --cc iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
index 690ce61,70e9b17..4b61af2
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/MGraph.java
@@@ -241,10 -240,10 +241,10 @@@ public class MGraph implements Serializ
public Metadata getMetadata() throws PathErrorException {
Map<String, List<MeasurementSchema>> seriesMap = getSchemaForAllType();
Map<String, List<String>> deviceIdMap = getDeviceForAllType();
- return new Metadata(seriesMap, deviceIdMap);
+ return new Metadata(deviceIdMap);
}
- public HashSet<String> getAllStorageGroup() throws PathErrorException {
+ public HashSet<String> getAllStorageGroup() {
return mtree.getAllStorageGroup();
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/metadata/Metadata.java
index 5c56fe1,70aa9e4..47296e9
--- a/iotdb/src/main/java/org/apache/iotdb/db/metadata/Metadata.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/metadata/Metadata.java
@@@ -131,13 -76,13 +76,13 @@@ public class Metadata implements Serial
@Override
public boolean equals(Object obj) {
-- if(this == obj){
++ if (this == obj) {
return true;
}
-- if(obj == null){
++ if (obj == null) {
return false;
}
-- if(this.getClass() != obj.getClass()){
++ if (this.getClass() != obj.getClass()) {
return false;
}
@@@ -167,7 -98,7 +98,8 @@@
/**
* only used to check if deviceIdMap is equal to another deviceIdMap
*/
-- private boolean deviceIdMapEquals(Map<String, List<String>> map1, Map<String, List<String>> map2) {
++ private boolean deviceIdMapEquals(Map<String, List<String>> map1,
++ Map<String, List<String>> map2) {
if (!map1.keySet().equals(map2.keySet())) {
return false;
}
diff --cc iotdb/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java
index 15bcb5a,920aeef..42a2e8b
--- a/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/qp/executor/IQueryProcessExecutor.java
@@@ -51,8 -56,7 +56,9 @@@ public interface IQueryProcessExecutor
QueryFilterOptimizationException, ProcessorException;
/**
-- * process aggregate plan of qp layer, construct queryDataSet.
++ * process aggregate plan of qp layer, construct queryDataSet. <<<<<<< HEAD
+ *
++ * ======= >>>>>>> master
*/
QueryDataSet aggregate(List<Path> paths, List<String> aggres, IExpression expression,
QueryContext context)
diff --cc iotdb/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByEngineDataSet.java
index cd1b95b,9cf4dc3..2f1b924
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByEngineDataSet.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/dataset/groupby/GroupByEngineDataSet.java
@@@ -16,7 -16,7 +16,6 @@@
* specific language governing permissions and limitations
* under the License.
*/
--
package org.apache.iotdb.db.query.dataset.groupby;
import java.util.ArrayList;
@@@ -24,9 -24,9 +23,9 @@@ import java.util.List
import org.apache.iotdb.db.exception.PathErrorException;
import org.apache.iotdb.db.exception.ProcessorException;
import org.apache.iotdb.db.metadata.MManager;
--import org.apache.iotdb.db.query.factory.AggreFuncFactory;
import org.apache.iotdb.db.query.aggregation.AggreResultData;
import org.apache.iotdb.db.query.aggregation.AggregateFunction;
++import org.apache.iotdb.db.query.factory.AggreFuncFactory;
import org.apache.iotdb.tsfile.exception.write.UnSupportedDataTypeException;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.Field;
diff --cc iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/AbstractNodeConstructor.java
index 712868f,5d83314..c23c2b9
--- a/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/AbstractNodeConstructor.java
+++ b/iotdb/src/main/java/org/apache/iotdb/db/query/timegenerator/AbstractNodeConstructor.java
@@@ -55,6 -58,7 +58,10 @@@ public abstract class AbstractNodeConst
/**
* Construct not series type node.
++<<<<<<< HEAD
++=======
+ *
++>>>>>>> master
* @param expression expression
* @return Node object
* @throws FileNodeManagerException FileNodeManagerException
diff --cc pom.xml
index 7b1fbfa,d8544b9..0a1a01c
--- a/pom.xml
+++ b/pom.xml
@@@ -135,9 -135,8 +135,9 @@@
<module>service-rpc</module>
<module>iotdb-cli</module>
<module>example</module>
- <!-- <module>hadoop</module> -->
+ <module>cluster</module>
+ <!--<module>hadoop</module>-->
- <!--<module>spark</module>-->
+ <module>spark</module>
</modules>
<build>
<pluginManagement>
diff --cc tsfile/src/main/java/org/apache/iotdb/tsfile/read/expression/ExpressionType.java
index 71e4c11,31fa6d3..5a36a1f
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/read/expression/ExpressionType.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/read/expression/ExpressionType.java
@@@ -19,5 -19,33 +19,32 @@@
package org.apache.iotdb.tsfile.read.expression;
public enum ExpressionType {
- AND, OR, SERIES, GLOBAL_TIME, TRUE
-
+ /**
+ * Represent the relationship between the left expression and the right expression is AND
+ */
+ AND,
+
+ /**
+ * Represent the relationship between the left expression and the right expression is OR
+ */
+ OR,
+
+ /**
+ * Represents that the expression is a leaf node in the expression tree and the type is value
+ * filtering
+ */
+ SERIES,
+
+ /**
+ * Represents that the expression is a leaf node in the expression tree and the type is time
+ * filtering
+ */
+ GLOBAL_TIME,
+
+ /**
+ * This type is used in the pruning process of expression tree in the distributed reading process.
+ * When pruning a expression tree for a data group, leaf nodes belonging to other data groups will
+ * be set to that type.
+ */
+ TRUE
}