You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by lt...@apache.org on 2019/11/08 06:59:10 UTC
[incubator-iotdb] 02/03: Merge branch 'master' into
add_hot_load_configuration
This is an automated email from the ASF dual-hosted git repository.
lta pushed a commit to branch add_hot_load_configuration
in repository https://gitbox.apache.org/repos/asf/incubator-iotdb.git
commit a2dc1659e9a98cbe6696c5b28246382b0496ba8e
Merge: bc27c55 efac7c4
Author: lta <li...@163.com>
AuthorDate: Fri Nov 8 09:48:32 2019 +0800
Merge branch 'master' into add_hot_load_configuration
.../UserGuide/3-Server/4-Config Manual.md | 27 +++
.../3-Writing Data on HDFS.md | 27 +++
.../UserGuide/3-Server/4-Config Manual.md | 28 +++
...e Hadoop Connector.md => 2-MapReduce TsFile.md} | 1 -
...TsFile Spark Connector.md => 3-Spark TsFile.md} | 0
...4-Spark IoTDB Connector.md => 4-Spark IoTDB.md} | 10 +-
...5-TsFile Hive Connector.md => 5-Hive TsFile.md} | 0
.../3-Writing Data on HDFS.md | 27 +++
.../iotdb/hadoop/fileSystem/HDFSConfUtil.java | 14 ++
.../apache/iotdb/jdbc/IoTDBPreparedStatement.java | 30 +--
.../java/org/apache/iotdb/jdbc/IoTDBStatement.java | 25 +-
.../test/java/org/apache/iotdb/jdbc/BatchTest.java | 29 ++-
.../resources/conf/iotdb-engine.properties | 12 +-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 40 ++++
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 27 ++-
.../org/apache/iotdb/db/service/TSServiceImpl.java | 149 +++++++-----
.../apache/iotdb/db/utils/TypeInferenceUtils.java | 10 +-
.../apache/iotdb/db/integration/IoTDBCloseIT.java | 202 ++++++++++++++++
service-rpc/rpc-changelist.md | 1 +
service-rpc/src/main/thrift/rpc.thrift | 3 +
.../java/org/apache/iotdb/session/Session.java | 44 ++--
.../iotdb/{sparkdb => spark/db}/SQLConstant.java | 2 +-
.../iotdb/{sparkdb => spark/db}/Converter.scala | 2 +-
.../{sparkdb => spark/db}/DefaultSource.scala | 2 +-
.../iotdb/{sparkdb => spark/db}/IoTDBOptions.scala | 2 +-
.../iotdb/{sparkdb => spark/db}/IoTDBRDD.scala | 8 +-
.../{sparkdb => spark/db}/IoTDBRelation.scala | 2 +-
.../iotdb/{sparkdb => spark/db}/Transformer.scala | 66 +++--
.../iotdb/{sparkdb => spark/db}/package.scala | 6 +-
.../{sparkdb => spark/db}/EnvironmentUtils.java | 2 +-
.../iotdb/{sparkdb => spark/db}/IoTDBTest.scala | 2 +-
.../apache/iotdb/spark/tsfile/DefaultSource.scala | 16 +-
.../apache/iotdb/spark/tsfile/Transformer.scala | 56 ++---
.../org/apache/iotdb/spark/tsfile/package.scala | 6 +-
.../org/apache/iotdb/spark/tsfile/TSFileSuit.scala | 4 +-
.../iotdb/tsfile/common/conf/TSFileConfig.java | 265 ++++++++++++---------
36 files changed, 798 insertions(+), 349 deletions(-)
diff --cc server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 28fcd97,8efc8d6..25091b3
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@@ -291,14 -342,57 +301,21 @@@ public class IoTDBDescriptor
Boolean.parseBoolean(properties.getProperty("dfs_ha_automatic_failover_enabled")));
TSFileDescriptor.getInstance().getConfig().setDfsClientFailoverProxyProvider(
properties.getProperty("dfs_client_failover_proxy_provider"));
+ TSFileDescriptor.getInstance().getConfig().setUseKerberos(Boolean.parseBoolean(
+ properties.getProperty("hdfs_use_kerberos", String.valueOf(conf.isUseKerberos()))));
+ TSFileDescriptor.getInstance().getConfig().setKerberosKeytabFilePath(
+ properties.getProperty("kerberos_keytab_file_path", conf.getKerberosKeytabFilePath()));
+ TSFileDescriptor.getInstance().getConfig().setKerberosPrincipal(
+ properties.getProperty("kerberos_principal", conf.getKerberosPrincipal()));
+
// set tsfile-format config
- TSFileDescriptor.getInstance().getConfig().setGroupSizeInByte(Integer
- .parseInt(properties.getProperty("group_size_in_byte",
- Integer.toString(TSFileDescriptor.getInstance().getConfig().getGroupSizeInByte()))));
- TSFileDescriptor.getInstance().getConfig().setPageSizeInByte(Integer
- .parseInt(properties.getProperty("page_size_in_byte",
- Integer.toString(TSFileDescriptor.getInstance().getConfig().getPageSizeInByte()))));
- if (TSFileDescriptor.getInstance().getConfig().getPageSizeInByte() > TSFileDescriptor
- .getInstance().getConfig().getGroupSizeInByte()) {
- logger
- .warn("page_size is greater than group size, will set it as the same with group size");
- TSFileDescriptor.getInstance().getConfig()
- .setPageSizeInByte(TSFileDescriptor.getInstance().getConfig().getGroupSizeInByte());
- }
- TSFileDescriptor.getInstance().getConfig().setMaxNumberOfPointsInPage(Integer
- .parseInt(properties.getProperty("max_number_of_points_in_page",
- Integer.toString(
- TSFileDescriptor.getInstance().getConfig().getMaxNumberOfPointsInPage()))));
- TSFileDescriptor.getInstance().getConfig().setTimeSeriesDataType(properties
- .getProperty("time_series_data_type",
- TSFileDescriptor.getInstance().getConfig().getTimeSeriesDataType()));
- TSFileDescriptor.getInstance().getConfig().setMaxStringLength(Integer
- .parseInt(properties.getProperty("max_string_length",
- Integer.toString(TSFileDescriptor.getInstance().getConfig().getMaxStringLength()))));
- TSFileDescriptor.getInstance().getConfig().setBloomFilterErrorRate(Double
- .parseDouble(properties.getProperty("bloom_filter_error_rate",
- Double.toString(
- TSFileDescriptor.getInstance().getConfig().getBloomFilterErrorRate()))));
- TSFileDescriptor.getInstance().getConfig().setFloatPrecision(Integer
- .parseInt(properties
- .getProperty("float_precision", Integer
- .toString(TSFileDescriptor.getInstance().getConfig().getFloatPrecision()))));
- TSFileDescriptor.getInstance().getConfig().setTimeEncoder(properties
- .getProperty("time_encoder",
- TSFileDescriptor.getInstance().getConfig().getTimeEncoder()));
- TSFileDescriptor.getInstance().getConfig().setValueEncoder(properties
- .getProperty("value_encoder",
- TSFileDescriptor.getInstance().getConfig().getValueEncoder()));
- TSFileDescriptor.getInstance().getConfig().setCompressor(properties
- .getProperty("compressor", TSFileDescriptor.getInstance().getConfig().getCompressor()));
+ loadTsFileProps(properties);
+ } catch (FileNotFoundException e) {
+ logger.warn("Fail to find config file {}", url, e);
} catch (IOException e) {
- logger.warn("Cannot load config file because, use default configuration", e);
+ logger.warn("Cannot load config file, use default configuration", e);
} catch (Exception e) {
logger.warn("Incorrect format in config file, use default configuration", e);
} finally {