You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iotdb.apache.org by he...@apache.org on 2023/05/23 07:49:16 UTC
[iotdb] branch tiered_storage updated: add config
This is an automated email from the ASF dual-hosted git repository.
heiming pushed a commit to branch tiered_storage
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/tiered_storage by this push:
new 9953765c368 add config
9953765c368 is described below
commit 9953765c36885ee624d1fd00dc4ebc38e77a2f0b
Author: HeimingZ <zh...@qq.com>
AuthorDate: Tue May 23 15:48:56 2023 +0800
add config
---
.../iotdb/hadoop/tsfile/TsFileWriteToHDFS.java | 2 +-
.../iotdb/hadoop/fileSystem/HDFSConfUtil.java | 11 ++-
.../apache/iotdb/hadoop/tsfile/TSFHadoopTest.java | 6 +-
.../apache/iotdb/hive/TSFHiveInputFormatTest.java | 4 +-
.../apache/iotdb/hive/TSFHiveRecordReaderTest.java | 4 +-
.../resources/conf/iotdb-common.properties | 21 ++++-
.../iotdb/os/HybridFileInputFactoryDecorator.java | 3 +-
.../apache/iotdb/os/conf/ObjectStorageConfig.java | 57 ++++++++++++--
.../os/{io/aws => conf/provider}/AWSS3Config.java | 27 +------
.../provider/OSProviderConfig.java} | 45 +++++++----
.../iotdb/os/io/aws/S3ObjectStorageConnector.java | 13 +++-
.../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 89 +++++++++++++++++++---
.../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 56 ++++++++++++--
.../iotdb/tsfile/common/conf/TSFileConfig.java | 6 +-
.../org/apache/iotdb/tsfile/utils/FSUtils.java | 16 +++-
.../apache/iotdb/tsfile/utils/FilePathUtils.java | 14 ++--
16 files changed, 281 insertions(+), 93 deletions(-)
diff --git a/example/hadoop/src/main/java/org/apache/iotdb/hadoop/tsfile/TsFileWriteToHDFS.java b/example/hadoop/src/main/java/org/apache/iotdb/hadoop/tsfile/TsFileWriteToHDFS.java
index 52e7fc670d7..aaf1619a400 100644
--- a/example/hadoop/src/main/java/org/apache/iotdb/hadoop/tsfile/TsFileWriteToHDFS.java
+++ b/example/hadoop/src/main/java/org/apache/iotdb/hadoop/tsfile/TsFileWriteToHDFS.java
@@ -43,7 +43,7 @@ public class TsFileWriteToHDFS {
private static final Logger logger = LoggerFactory.getLogger(TsFileWriteToHDFS.class);
public static void main(String[] args) {
- config.setTSFileStorageFs(FSType.HDFS);
+ config.setTSFileStorageFs(new FSType[] {FSType.HDFS});
String path = "hdfs://localhost:9000/test.tsfile";
File f = FSFactoryProducer.getFSFactory().getFile(path);
diff --git a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
index 40cefcf83be..d18c123ad5c 100644
--- a/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
+++ b/hadoop/src/main/java/org/apache/iotdb/hadoop/fileSystem/HDFSConfUtil.java
@@ -39,9 +39,18 @@ class HDFSConfUtil {
private static final Logger logger = LoggerFactory.getLogger(HDFSConfUtil.class);
static Configuration setConf(Configuration conf) {
- if (!tsFileConfig.getTSFileStorageFs().equals(FSType.HDFS)) {
+ boolean enableHDFS = false;
+ for (FSType type : tsFileConfig.getTSFileStorageFs()) {
+ if (type.equals(FSType.HDFS)) {
+ enableHDFS = true;
+ break;
+ }
+ }
+
+ if (!enableHDFS) {
return conf;
}
+
try {
conf.addResource(new File(tsFileConfig.getCoreSitePath()).toURI().toURL());
conf.addResource(new File(tsFileConfig.getHdfsSitePath()).toURI().toURL());
diff --git a/hadoop/src/test/java/org/apache/iotdb/hadoop/tsfile/TSFHadoopTest.java b/hadoop/src/test/java/org/apache/iotdb/hadoop/tsfile/TSFHadoopTest.java
index b86e8123b74..3307ec01bc9 100644
--- a/hadoop/src/test/java/org/apache/iotdb/hadoop/tsfile/TSFHadoopTest.java
+++ b/hadoop/src/test/java/org/apache/iotdb/hadoop/tsfile/TSFHadoopTest.java
@@ -71,7 +71,7 @@ public class TSFHadoopTest {
.concat(File.separator)
.concat("1-0-0-0.tsfile");
- private FSType beforeFSType;
+ private FSType[] beforeFSType;
@Before
public void setUp() {
@@ -152,7 +152,7 @@ public class TSFHadoopTest {
TSFInputFormat.setInputPaths(job, tsfilePath);
List<InputSplit> inputSplits = inputFormat.getSplits(job);
beforeFSType = TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs();
- TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(FSType.HDFS);
+ TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(new FSType[] {FSType.HDFS});
TsFileSequenceReader reader =
new TsFileSequenceReader(new HDFSInput(tsfilePath, job.getConfiguration()));
System.out.println(reader.readFileMetadata());
@@ -184,7 +184,7 @@ public class TSFHadoopTest {
TSFInputFormat.setReadTime(job, false);
List<InputSplit> inputSplits = inputFormat.getSplits(job);
beforeFSType = TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs();
- TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(FSType.HDFS);
+ TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(new FSType[] {FSType.HDFS});
TsFileSequenceReader reader =
new TsFileSequenceReader(new HDFSInput(tsfilePath, job.getConfiguration()));
diff --git a/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java b/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java
index 57d1c4ffc51..031a1b5b8a1 100644
--- a/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java
+++ b/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveInputFormatTest.java
@@ -46,7 +46,7 @@ public class TSFHiveInputFormatTest {
private TSFInputSplit inputSplit;
private TSFHiveInputFormat inputFormat;
private JobConf job;
- private FSType beforeFSType;
+ private FSType[] beforeFSType;
private final String filePath =
TestConstant.BASE_OUTPUT_PATH
.concat("data")
@@ -75,7 +75,7 @@ public class TSFHiveInputFormatTest {
String[] hosts = {"127.0.0.1"};
inputSplit = new TSFInputSplit(path, hosts, 0, 3727688L);
beforeFSType = TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs();
- TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(FSType.HDFS);
+ TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(new FSType[] {FSType.HDFS});
}
@After
diff --git a/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveRecordReaderTest.java b/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveRecordReaderTest.java
index 8480dcd0ae1..92c20e2c62e 100644
--- a/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveRecordReaderTest.java
+++ b/hive-connector/src/test/java/org/apache/iotdb/hive/TSFHiveRecordReaderTest.java
@@ -61,7 +61,7 @@ public class TSFHiveRecordReaderTest {
.concat("0")
.concat(File.separator)
.concat("1-0-0-0.tsfile");
- private FSType beforeFSType;
+ private FSType[] beforeFSType;
@Before
public void setUp() throws IOException {
@@ -70,7 +70,7 @@ public class TSFHiveRecordReaderTest {
Path path = new Path(filePath);
String[] hosts = {"127.0.0.1"};
beforeFSType = TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs();
- TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(FSType.HDFS);
+ TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(new FSType[] {FSType.HDFS});
TSFInputSplit inputSplit = new TSFInputSplit(path, hosts, 0, 3727528L);
String[] deviceIds = {"device_1"}; // configure reading which deviceIds
job.set(READ_DELTAOBJECTS, String.join(",", deviceIds));
diff --git a/node-commons/src/assembly/resources/conf/iotdb-common.properties b/node-commons/src/assembly/resources/conf/iotdb-common.properties
index 6e7cb3f7371..f688c00509f 100644
--- a/node-commons/src/assembly/resources/conf/iotdb-common.properties
+++ b/node-commons/src/assembly/resources/conf/iotdb-common.properties
@@ -1173,14 +1173,29 @@ cluster_name=defaultCluster
### Object storage management
####################
+# Datatype: boolean
+# enable_hdfs=false
+
+# Datatype: boolean
+# enable_object_storage=false
+
+# Datatype: string
+# remote_tsfile_cache_dirs=data/datanode/remote/cache
+
+# Datatype: long
+# remote_tsfile_cache_max_disk_usage=53687091200
+
+# Datatype: int
+# remote_tsfile_cache_page_size=20971520
+
# Datatype: string
-# object_storage_name=aws_s3
+# object_storage_name=AWS_S3
# Datatype: string
-# object_storage_bucket=iotdb
+# object_storage_endpoint=yourEndpoint
# Datatype: string
-# object_storage_endpoiont=yourEndpoint
+# object_storage_bucket=iotdb
# Datatype: string
# object_storage_access_key=yourAccessKey
diff --git a/object-storage/src/main/java/org/apache/iotdb/os/HybridFileInputFactoryDecorator.java b/object-storage/src/main/java/org/apache/iotdb/os/HybridFileInputFactoryDecorator.java
index 2f7e425ca60..12f0d05710c 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/HybridFileInputFactoryDecorator.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/HybridFileInputFactoryDecorator.java
@@ -20,7 +20,6 @@ package org.apache.iotdb.os;
import org.apache.iotdb.os.conf.ObjectStorageConfig;
import org.apache.iotdb.os.conf.ObjectStorageDescriptor;
-import org.apache.iotdb.os.io.aws.AWSS3Config;
import org.apache.iotdb.tsfile.fileSystem.fileInputFactory.FileInputFactory;
import org.apache.iotdb.tsfile.fileSystem.fileInputFactory.HybridFileInputFactory;
import org.apache.iotdb.tsfile.read.reader.TsFileInput;
@@ -50,7 +49,7 @@ public class HybridFileInputFactoryDecorator implements FileInputFactory {
File file = new File(filePath);
if (!file.exists()) {
fileInputFactory.getTsFileInput(
- FSUtils.parseLocalTsFile2OSFile(file, AWSS3Config.getBucketName(), dataNodeId).getPath());
+ FSUtils.parseLocalTsFile2OSFile(file, config.getBucketName(), dataNodeId).getPath());
}
return fileInputFactory.getTsFileInput(filePath);
}
diff --git a/object-storage/src/main/java/org/apache/iotdb/os/conf/ObjectStorageConfig.java b/object-storage/src/main/java/org/apache/iotdb/os/conf/ObjectStorageConfig.java
index 23724ca677f..4f9d49d41c1 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/conf/ObjectStorageConfig.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/conf/ObjectStorageConfig.java
@@ -18,23 +18,25 @@
*/
package org.apache.iotdb.os.conf;
-import org.apache.iotdb.os.io.aws.AWSS3Config;
+import org.apache.iotdb.os.conf.provider.AWSS3Config;
+import org.apache.iotdb.os.conf.provider.OSProviderConfig;
import org.apache.iotdb.os.utils.ObjectStorageType;
import java.io.File;
+import java.util.Objects;
public class ObjectStorageConfig {
private ObjectStorageType osType = ObjectStorageType.AWS_S3;
- private AWSS3Config awss3Config = new AWSS3Config();
+ private OSProviderConfig providerConfig = new AWSS3Config();
private String[] cacheDirs = {
"data" + File.separator + "datanode" + File.separator + "data" + File.separator + "cache"
};
- private long cacheMaxDiskUsage = 20 * 1024 * 1024 * 1024L;
+ private long cacheMaxDiskUsage = 50 * 1024 * 1024 * 1024L;
- private int cachePageSize = 10 * 1024 * 1024;
+ private int cachePageSize = 20 * 1024 * 1024;
ObjectStorageConfig() {}
@@ -44,14 +46,51 @@ public class ObjectStorageConfig {
public void setOsType(ObjectStorageType osType) {
this.osType = osType;
+ if (Objects.requireNonNull(osType) == ObjectStorageType.AWS_S3) {
+ this.providerConfig = new AWSS3Config();
+ } else {
+ this.providerConfig = null;
+ }
}
- public String[] getCacheDirs() {
- return cacheDirs;
+ public OSProviderConfig getProviderConfig() {
+ return providerConfig;
+ }
+
+ public String getEndpoint() {
+ return providerConfig.getEndpoint();
+ }
+
+ public void setEndpoint(String endpoint) {
+ providerConfig.setEndpoint(endpoint);
}
public String getBucketName() {
- return AWSS3Config.getBucketName();
+ return providerConfig.getBucketName();
+ }
+
+ public void setBucketName(String bucketName) {
+ providerConfig.setBucketName(bucketName);
+ }
+
+ public String getAccessKeyId() {
+ return providerConfig.getAccessKeyId();
+ }
+
+ public void setAccessKeyId(String accessKeyId) {
+ providerConfig.setAccessKeyId(accessKeyId);
+ }
+
+ public String getAccessKeySecret() {
+ return providerConfig.getAccessKeySecret();
+ }
+
+ public void setAccessKeySecret(String accessKeySecret) {
+ providerConfig.setAccessKeySecret(accessKeySecret);
+ }
+
+ public String[] getCacheDirs() {
+ return cacheDirs;
}
public void setCacheDirs(String[] cacheDirs) {
@@ -62,6 +101,10 @@ public class ObjectStorageConfig {
return cacheMaxDiskUsage;
}
+ public void setCacheMaxDiskUsage(long cacheMaxDiskUsage) {
+ this.cacheMaxDiskUsage = cacheMaxDiskUsage;
+ }
+
public int getCachePageSize() {
return cachePageSize;
}
diff --git a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/AWSS3Config.java b/object-storage/src/main/java/org/apache/iotdb/os/conf/provider/AWSS3Config.java
similarity index 52%
copy from object-storage/src/main/java/org/apache/iotdb/os/io/aws/AWSS3Config.java
copy to object-storage/src/main/java/org/apache/iotdb/os/conf/provider/AWSS3Config.java
index 89b69cd5a2c..ff684bab7d0 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/AWSS3Config.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/conf/provider/AWSS3Config.java
@@ -17,29 +17,6 @@
* under the License.
*/
-package org.apache.iotdb.os.io.aws;
+package org.apache.iotdb.os.conf.provider;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
-
-public class AWSS3Config {
-
- private static String region;
- private static String bucketName;
- private static String access_key_id;
- private static String access_key_secret;
-
- public static String getRegion() {
- return region;
- }
-
- public static String getBucketName() {
- return bucketName;
- }
-
- public static AwsCredentialsProvider getCredentialProvider() {
- return StaticCredentialsProvider.create(
- AwsBasicCredentials.create(access_key_id, access_key_secret));
- }
-}
+public class AWSS3Config extends OSProviderConfig {}
diff --git a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/AWSS3Config.java b/object-storage/src/main/java/org/apache/iotdb/os/conf/provider/OSProviderConfig.java
similarity index 51%
rename from object-storage/src/main/java/org/apache/iotdb/os/io/aws/AWSS3Config.java
rename to object-storage/src/main/java/org/apache/iotdb/os/conf/provider/OSProviderConfig.java
index 89b69cd5a2c..9c42c673a99 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/AWSS3Config.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/conf/provider/OSProviderConfig.java
@@ -16,30 +16,43 @@
* specific language governing permissions and limitations
* under the License.
*/
+package org.apache.iotdb.os.conf.provider;
-package org.apache.iotdb.os.io.aws;
+public abstract class OSProviderConfig {
+ protected String endpoint;
+ protected String bucketName;
+ protected String accessKeyId;
+ protected String accessKeySecret;
-import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
-import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
-import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+ public String getEndpoint() {
+ return endpoint;
+ }
-public class AWSS3Config {
+ public void setEndpoint(String endpoint) {
+ this.endpoint = endpoint;
+ }
- private static String region;
- private static String bucketName;
- private static String access_key_id;
- private static String access_key_secret;
+ public String getBucketName() {
+ return bucketName;
+ }
- public static String getRegion() {
- return region;
+ public void setBucketName(String bucketName) {
+ this.bucketName = bucketName;
}
- public static String getBucketName() {
- return bucketName;
+ public String getAccessKeyId() {
+ return accessKeyId;
+ }
+
+ public void setAccessKeyId(String accessKeyId) {
+ this.accessKeyId = accessKeyId;
+ }
+
+ public String getAccessKeySecret() {
+ return accessKeySecret;
}
- public static AwsCredentialsProvider getCredentialProvider() {
- return StaticCredentialsProvider.create(
- AwsBasicCredentials.create(access_key_id, access_key_secret));
+ public void setAccessKeySecret(String accessKeySecret) {
+ this.accessKeySecret = accessKeySecret;
}
}
diff --git a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java b/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java
index 24416857382..80753b2af2e 100644
--- a/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java
+++ b/object-storage/src/main/java/org/apache/iotdb/os/io/aws/S3ObjectStorageConnector.java
@@ -19,11 +19,15 @@
package org.apache.iotdb.os.io.aws;
+import org.apache.iotdb.os.conf.ObjectStorageDescriptor;
+import org.apache.iotdb.os.conf.provider.AWSS3Config;
import org.apache.iotdb.os.exception.ObjectStorageException;
import org.apache.iotdb.os.fileSystem.OSURI;
import org.apache.iotdb.os.io.IMetaData;
import org.apache.iotdb.os.io.ObjectStorageConnector;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.ResponseBytes;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
@@ -45,10 +49,15 @@ import java.io.InputStream;
public class S3ObjectStorageConnector implements ObjectStorageConnector {
private static final String RANGE_FORMAT = "%d-%d";
+ private static final AWSS3Config s3config =
+ (AWSS3Config) ObjectStorageDescriptor.getInstance().getConfig().getProviderConfig();
private static final S3Client s3Client =
S3Client.builder()
- .region(Region.of(AWSS3Config.getRegion()))
- .credentialsProvider(AWSS3Config.getCredentialProvider())
+ .region(Region.of(s3config.getEndpoint()))
+ .credentialsProvider(
+ StaticCredentialsProvider.create(
+ AwsBasicCredentials.create(
+ s3config.getAccessKeyId(), s3config.getAccessKeySecret())))
.build();
@Override
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
index 406f1dd4d09..a3ace6203de 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -39,6 +39,9 @@ import org.apache.iotdb.db.service.thrift.impl.ClientRPCServiceImpl;
import org.apache.iotdb.db.service.thrift.impl.NewInfluxDBServiceImpl;
import org.apache.iotdb.db.utils.datastructure.TVListSortAlgorithm;
import org.apache.iotdb.db.wal.utils.WALMode;
+import org.apache.iotdb.os.conf.ObjectStorageConfig;
+import org.apache.iotdb.os.conf.ObjectStorageDescriptor;
+import org.apache.iotdb.os.utils.ObjectStorageType;
import org.apache.iotdb.rpc.RpcTransportFactory;
import org.apache.iotdb.rpc.RpcUtils;
import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
@@ -725,6 +728,9 @@ public class IoTDBConfig {
/** Default TSfile storage is in local file system */
private FSType tsFileStorageFs = FSType.LOCAL;
+ /** Enable hdfs or not */
+ private boolean enableHDFS = false;
+
/** Default core-site.xml file path is /etc/hadoop/conf/core-site.xml */
private String coreSitePath = "/etc/hadoop/conf/core-site.xml";
@@ -1129,13 +1135,14 @@ public class IoTDBConfig {
*/
private String RateLimiterType = "FixedIntervalRateLimiter";
+ /** Threads for migration tasks */
private int migrateThreadCount = 3;
- private String objectStorageName = "aws_s3";
- private String objectStorageBucket = "iotdb";
- private String objectStorageEndpoiont = "yourEndpoint";
- private String objectStorageAccessKey = "yourAccessKey";
- private String objectStorageAccessSecret = "yourAccessSecret";
+ /** Enable hdfs or not */
+ private boolean enableObjectStorage = false;
+
+ /** Config for object storage */
+ private ObjectStorageConfig osConfig = ObjectStorageDescriptor.getInstance().getConfig();
IoTDBConfig() {}
@@ -2447,6 +2454,14 @@ public class IoTDBConfig {
this.tsFileStorageFs = FSType.valueOf(tsFileStorageFs);
}
+ public boolean isEnableHDFS() {
+ return enableHDFS;
+ }
+
+ public void setEnableHDFS(boolean enableHDFS) {
+ this.enableHDFS = enableHDFS;
+ }
+
String getCoreSitePath() {
return coreSitePath;
}
@@ -3917,24 +3932,76 @@ public class IoTDBConfig {
this.migrateThreadCount = migrateThreadCount;
}
+ public boolean isEnableObjectStorage() {
+ return enableObjectStorage;
+ }
+
+ public void setEnableObjectStorage(boolean enableObjectStorage) {
+ this.enableObjectStorage = enableObjectStorage;
+ }
+
public String getObjectStorageName() {
- return objectStorageName;
+ return osConfig.getOsType().name();
+ }
+
+ public void setObjectStorageName(String objectStorageName) {
+ osConfig.setOsType(ObjectStorageType.valueOf(objectStorageName));
}
public String getObjectStorageBucket() {
- return objectStorageBucket;
+ return osConfig.getBucketName();
+ }
+
+ public void setObjectStorageBucket(String objectStorageBucket) {
+ osConfig.setBucketName(objectStorageBucket);
+ }
+
+ public String getObjectStorageEndpoint() {
+ return osConfig.getEndpoint();
}
- public String getObjectStorageEndpoiont() {
- return objectStorageEndpoiont;
+ public void setObjectStorageEndpoint(String objectStorageEndpoint) {
+ osConfig.setEndpoint(objectStorageEndpoint);
}
public String getObjectStorageAccessKey() {
- return objectStorageAccessKey;
+ return osConfig.getAccessKeyId();
+ }
+
+ public void setObjectStorageAccessKey(String objectStorageAccessKey) {
+ osConfig.setAccessKeyId(objectStorageAccessKey);
}
public String getObjectStorageAccessSecret() {
- return objectStorageAccessSecret;
+ return osConfig.getAccessKeySecret();
+ }
+
+ public void setObjectStorageAccessSecret(String objectStorageAccessSecret) {
+ osConfig.setAccessKeySecret(objectStorageAccessSecret);
+ }
+
+ public String[] getCacheDirs() {
+ return osConfig.getCacheDirs();
+ }
+
+ public void setCacheDirs(String[] cacheDirs) {
+ osConfig.setCacheDirs(cacheDirs);
+ }
+
+ public long getCacheMaxDiskUsage() {
+ return osConfig.getCacheMaxDiskUsage();
+ }
+
+ public void setCacheMaxDiskUsage(long cacheMaxDiskUsage) {
+ osConfig.setCacheMaxDiskUsage(cacheMaxDiskUsage);
+ }
+
+ public int getCachePageSize() {
+ return osConfig.getCachePageSize();
+ }
+
+ public void setCachePageSize(int cachePageSize) {
+ osConfig.setCachePageSize(cachePageSize);
}
public String getClusterSchemaLimitLevel() {
diff --git a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
index 595b7d71915..6c5987cf955 100644
--- a/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
+++ b/server/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -66,6 +66,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.net.MalformedURLException;
import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Properties;
import java.util.ServiceLoader;
@@ -772,6 +774,9 @@ public class IoTDBDescriptor {
conf.setTsFileStorageFs(
properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().toString()));
+ conf.setEnableHDFS(
+ Boolean.parseBoolean(
+ properties.getProperty("enable_hdfs", String.valueOf(conf.isEnableHDFS()))));
conf.setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath()));
conf.setHdfsSitePath(properties.getProperty("hdfs_site_path", conf.getHdfsSitePath()));
conf.setHdfsIp(properties.getProperty("hdfs_ip", conf.getRawHDFSIp()).split(","));
@@ -967,11 +972,18 @@ public class IoTDBDescriptor {
conf.setExtPipeDir(properties.getProperty("ext_pipe_dir", conf.getExtPipeDir()).trim());
// At the same time, set TSFileConfig
- TSFileDescriptor.getInstance()
- .getConfig()
- .setTSFileStorageFs(
- FSType.valueOf(
- properties.getProperty("tsfile_storage_fs", conf.getTsFileStorageFs().name())));
+ List<FSType> fsTypes = new ArrayList<>();
+ fsTypes.add(FSType.LOCAL);
+ if (Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_object_storage", String.valueOf(conf.isEnableObjectStorage())))) {
+ fsTypes.add(FSType.OBJECT_STORAGE);
+ }
+ if (Boolean.parseBoolean(
+ properties.getProperty("enable_hdfs", String.valueOf(conf.isEnableHDFS())))) {
+ fsTypes.add(FSType.HDFS);
+ }
+ TSFileDescriptor.getInstance().getConfig().setTSFileStorageFs(fsTypes.toArray(new FSType[0]));
TSFileDescriptor.getInstance()
.getConfig()
.setCoreSitePath(properties.getProperty("core_site_path", conf.getCoreSitePath()));
@@ -1070,6 +1082,9 @@ public class IoTDBDescriptor {
// author cache
loadAuthorCache(properties);
+ // object storage
+ loadObjectStorageProps(properties);
+
conf.setTimePartitionInterval(
DateTimeUtils.convertMilliTimeWithPrecision(
conf.getTimePartitionInterval(), conf.getTimestampPrecision()));
@@ -1138,6 +1153,37 @@ public class IoTDBDescriptor {
"author_cache_expire_time", String.valueOf(conf.getAuthorCacheExpireTime()))));
}
+ private void loadObjectStorageProps(Properties properties) {
+ conf.setEnableObjectStorage(
+ Boolean.parseBoolean(
+ properties.getProperty(
+ "enable_object_storage", String.valueOf(conf.isEnableObjectStorage()))));
+ conf.setCacheDirs(
+ properties
+ .getProperty("remote_tsfile_cache_dirs", String.join(",", conf.getCacheDirs()))
+ .split(","));
+ conf.setCacheMaxDiskUsage(
+ Long.parseLong(
+ properties.getProperty(
+ "remote_tsfile_cache_max_disk_usage",
+ String.valueOf(conf.getCacheMaxDiskUsage()))));
+ conf.setCachePageSize(
+ Integer.parseInt(
+ properties.getProperty(
+ "remote_tsfile_cache_page_size", String.valueOf(conf.getCachePageSize()))));
+ conf.setObjectStorageName(
+ properties.getProperty("object_storage_name", conf.getObjectStorageName()));
+ conf.setObjectStorageEndpoint(
+ properties.getProperty("object_storage_endpoint", conf.getObjectStorageEndpoint()));
+ conf.setObjectStorageBucket(
+ properties.getProperty("object_storage_bucket", conf.getObjectStorageBucket()));
+ conf.setObjectStorageAccessKey(
+ properties.getProperty("object_storage_access_key", conf.getObjectStorageAccessKey()));
+ conf.setObjectStorageAccessSecret(
+ properties.getProperty(
+ "object_storage_access_secret", conf.getObjectStorageAccessSecret()));
+ }
+
private void loadWALProps(Properties properties) {
conf.setWalMode(
WALMode.valueOf((properties.getProperty("wal_mode", conf.getWalMode().toString()))));
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
index 4dd967556c8..cca02389716 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/common/conf/TSFileConfig.java
@@ -119,7 +119,7 @@ public class TSFileConfig implements Serializable {
/** Default endian value is BIG_ENDIAN. */
private String endian = "BIG_ENDIAN";
/** Default storage is in local file system */
- private FSType TSFileStorageFs = FSType.LOCAL;
+ private FSType[] TSFileStorageFs = FSType.values();
/** Default core-site.xml file path is /etc/hadoop/conf/core-site.xml */
private String coreSitePath = "/etc/hadoop/conf/core-site.xml";
/** Default hdfs-site.xml file path is /etc/hadoop/conf/hdfs-site.xml */
@@ -348,11 +348,11 @@ public class TSFileConfig implements Serializable {
this.bloomFilterErrorRate = bloomFilterErrorRate;
}
- public FSType getTSFileStorageFs() {
+ public FSType[] getTSFileStorageFs() {
return this.TSFileStorageFs;
}
- public void setTSFileStorageFs(FSType fileStorageFs) {
+ public void setTSFileStorageFs(FSType[] fileStorageFs) {
this.TSFileStorageFs = fileStorageFs;
}
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java
index 7133841052c..4897d75b441 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FSUtils.java
@@ -18,6 +18,7 @@
*/
package org.apache.iotdb.tsfile.utils;
+import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.fileSystem.FSPath;
import org.apache.iotdb.tsfile.fileSystem.FSType;
@@ -36,17 +37,28 @@ public class FSUtils {
private static final String[] fsFileClassName = {
"org.apache.iotdb.os.fileSystem.OSFile", "org.apache.iotdb.hadoop.fileSystem.HDFSFile"
};
+ private static final boolean[] isSupported = new boolean[fsTypes.length];
private static final Class<?>[] fsFileClass = new Class[fsTypes.length];
private FSUtils() {}
static {
+ for (FSType fsType : TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs()) {
+ if (fsType == FSType.OBJECT_STORAGE) {
+ isSupported[0] = true;
+ } else if (fsType == FSType.HDFS) {
+ isSupported[1] = true;
+ }
+ }
+
for (int i = 0; i < fsTypes.length; ++i) {
+ if (!isSupported[i]) {
+ continue;
+ }
try {
fsFileClass[i] = Class.forName(fsFileClassName[i]);
} catch (ClassNotFoundException e) {
- // TODO
- logger.info(
+ logger.error(
"Failed to get "
+ fsTypes[i].name()
+ " file system. Please check your dependency of "
diff --git a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
index 3d481be7c97..57b7a3b622f 100644
--- a/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
+++ b/tsfile/src/main/java/org/apache/iotdb/tsfile/utils/FilePathUtils.java
@@ -19,9 +19,7 @@
package org.apache.iotdb.tsfile.utils;
-import org.apache.iotdb.tsfile.common.conf.TSFileDescriptor;
import org.apache.iotdb.tsfile.common.constant.TsFileConstant;
-import org.apache.iotdb.tsfile.fileSystem.FSType;
import java.io.File;
@@ -29,11 +27,7 @@ import static org.apache.iotdb.tsfile.common.constant.TsFileConstant.TSFILE_SUFF
public class FilePathUtils {
- private static final String PATH_SPLIT_STRING =
- TSFileDescriptor.getInstance().getConfig().getTSFileStorageFs() == FSType.LOCAL
- && "\\".equals(File.separator)
- ? "\\\\"
- : "/";
+ private static final String LOCAL_PATH_SPLIT_STRING = "\\".equals(File.separator) ? "\\\\" : "/";
public static final String FILE_NAME_SEPARATOR = "-";
private FilePathUtils() {
@@ -64,7 +58,11 @@ public class FilePathUtils {
* @param tsFileAbsolutePath the tsFile Absolute Path
*/
public static String[] splitTsFilePath(String tsFileAbsolutePath) {
- return tsFileAbsolutePath.split(PATH_SPLIT_STRING);
+ String separator = LOCAL_PATH_SPLIT_STRING;
+ if (!FSUtils.isLocal(tsFileAbsolutePath)) {
+ separator = "/";
+ }
+ return tsFileAbsolutePath.split(separator);
}
public static String getLogicalStorageGroupName(String tsFileAbsolutePath) {