You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/11/16 10:49:18 UTC
[ambari-logsearch] branch master updated: AMBARI-24833. Extend
cloud log rolling options (#25)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new 31c70d3 AMBARI-24833. Extend cloud log rolling options (#25)
31c70d3 is described below
commit 31c70d37c30488205d91f666316949d9a7242dd8
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Fri Nov 16 11:49:15 2018 +0100
AMBARI-24833. Extend cloud log rolling options (#25)
* AMBARI-24833. Extend cloud log rolling options
* Add some more fixes, use default-fs if hdfs-client is used with cloud storage.
* AMBARI-24833. Overide hdfs username, if core hdfs is used as fs
* AMBARI-24833. Remove aws-java-sdk-bundle dependency (~100MB)
* AMBARI-24833. change size format to size unit
---
ambari-logsearch-logfeeder/pom.xml | 11 +++++
.../logfeeder/common/LogFeederConstants.java | 9 ++--
.../logfeeder/conf/CloudStorageDestination.java | 2 +-
.../ambari/logfeeder/conf/LogFeederProps.java | 20 +++++++-
.../conf/output/ExternalHdfsOutputConfig.java | 17 -------
.../logfeeder/conf/output/RolloverConfig.java | 54 ++++++++++++++++++----
.../output/cloud/CloudStorageLoggerFactory.java | 9 +++-
.../output/cloud/CloudStorageUploader.java | 2 +-
.../cloud/upload/ExternalHDFSUploadClient.java | 6 +--
.../output/cloud/upload/HDFSUploadClient.java | 12 ++++-
.../output/cloud/upload/UploadClientFactory.java | 1 +
.../src/main/resources/logfeeder.properties | 4 +-
12 files changed, 105 insertions(+), 42 deletions(-)
diff --git a/ambari-logsearch-logfeeder/pom.xml b/ambari-logsearch-logfeeder/pom.xml
index e71b3cc..003b233 100644
--- a/ambari-logsearch-logfeeder/pom.xml
+++ b/ambari-logsearch-logfeeder/pom.xml
@@ -150,6 +150,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-aws</artifactId>
<version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-bundle</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.amazonaws</groupId>
@@ -162,6 +168,11 @@
<version>${aws-sdk.version}</version>
</dependency>
<dependency>
+ <groupId>com.amazonaws</groupId>
+ <artifactId>aws-java-sdk-dynamodb</artifactId>
+ <version>${aws-sdk.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-azure</artifactId>
<version>${hadoop.version}</version>
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
index 11d351f..b5fffa8 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/common/LogFeederConstants.java
@@ -34,10 +34,6 @@ public class LogFeederConstants {
// Event History Constants History
public static final String VALUES = "jsons";
public static final String ROW_TYPE = "rowtype";
-
- // S3 Constants
- public static final String S3_PATH_START_WITH = "s3://";
- public static final String S3_PATH_SEPARATOR = "/";
public static final String IN_MEMORY_TIMESTAMP = "in_memory_timestamp";
@@ -122,14 +118,17 @@ public class LogFeederConstants {
public static final String CLOUD_ROLLOVER_ARCHIVE_LOCATION = "logfeeder.cloud.rollover.archive.base.dir";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_MIN = "logfeeder.cloud.rollover.threshold.min";
public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE = "logfeeder.cloud.rollover.threshold.size";
+ public static final String CLOUD_ROLLOVER_MAX_BACKUP_FILES = "logfeeder.cloud.rollover.max.files";
+ public static final String CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE_UNIT = "logfeeder.cloud.rollover.threshold.size.unit";
public static final String CLOUD_ROLLOVER_USE_GZIP = "logfeeder.cloud.rollover.use.gzip";
public static final String CLOUD_ROLLOVER_IMMEDIATE_FLUSH = "logfeeder.cloud.rollover.immediate.flush";
public static final String CLOUD_ROLLOVER_ON_SHUTDOWN = "logfeeder.cloud.rollover.on.shutdown";
public static final String CLOUD_ROLLOVER_ON_STARTUP = "logfeeder.cloud.rollover.on.startup";
+ public static final String HDFS_USER = "logfeeder.hdfs.user";
+
public static final String HDFS_HOST = "logfeeder.hdfs.host";
public static final String HDFS_PORT = "logfeeder.hdfs.port";
- public static final String HDFS_USER = "logfeeder.hdfs.user";
public static final String HDFS_FILE_PERMISSIONS = "logfeeder.hdfs.file.permissions";
public static final String HDFS_KERBEROS = "logfeeder.hdfs.kerberos";
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java
index 1a7eafa..0246b98 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/CloudStorageDestination.java
@@ -19,7 +19,7 @@
package org.apache.ambari.logfeeder.conf;
public enum CloudStorageDestination {
- HDFS("hdfs"), S3("s3"), GCS("gcs"), ADLS("adls"), NONE("none");
+ HDFS("hdfs"), S3("s3"), GCS("gcs"), ADLS("adls"), WASB("wasb"), DEFAULT_FS("default-fs"), NONE("none");
private String text;
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
index d32e1df..83f10e4 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/LogFeederProps.java
@@ -224,7 +224,8 @@ public class LogFeederProps implements LogFeederProperties {
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_STORAGE_DESTINATION,
description = "Type of storage that is the destination for cloud output logs.",
- examples = {"hdfs", "s3", "gcs", "adls", "none"},
+ examples = {"hdfs", "s3", "gcs", "adls", "wasb", "none"},
+ defaultValue = "none",
sources = {LogFeederConstants.CLOUD_STORAGE_DESTINATION}
)
@Value("${" + LogFeederConstants.CLOUD_STORAGE_DESTINATION + ":none}")
@@ -279,6 +280,15 @@ public class LogFeederProps implements LogFeederProperties {
@Value("${" + LogFeederConstants.CLOUD_STORAGE_BASE_PATH + ":}")
private String cloudBasePath;
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.HDFS_USER,
+ description = "Overrides HADOOP_USER_NAME variable at runtime",
+ examples = {"hdfs"},
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.HDFS_USER + ":}")
+ private String logfeederHdfsUser;
+
@Inject
private LogEntryCacheConfig logEntryCacheConfig;
@@ -492,6 +502,14 @@ public class LogFeederProps implements LogFeederProperties {
return useCloudHdfsClient;
}
+ public String getLogfeederHdfsUser() {
+ return logfeederHdfsUser;
+ }
+
+ public void setLogfeederHdfsUser(String logfeederHdfsUser) {
+ this.logfeederHdfsUser = logfeederHdfsUser;
+ }
+
public void setUseCloudHdfsClient(boolean useCloudHdfsClient) {
this.useCloudHdfsClient = useCloudHdfsClient;
}
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/ExternalHdfsOutputConfig.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/ExternalHdfsOutputConfig.java
index 70772f7..fbbf869 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/ExternalHdfsOutputConfig.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/ExternalHdfsOutputConfig.java
@@ -55,15 +55,6 @@ public class ExternalHdfsOutputConfig {
private String hdfsFilePermissions;
@LogSearchPropertyDescription(
- name = LogFeederConstants.HDFS_USER,
- description = "Overrides HADOOP_USER_NAME variable at runtime",
- examples = {"hdfs"},
- sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
- )
- @Value("${"+ LogFeederConstants.HDFS_USER + ":}")
- private String hdfsUser;
-
- @LogSearchPropertyDescription(
name = LogFeederConstants.HDFS_KERBEROS,
description = "Enable kerberos support for HDFS",
examples = {"true"},
@@ -97,14 +88,6 @@ public class ExternalHdfsOutputConfig {
this.hdfsFilePermissions = hdfsFilePermissions;
}
- public String getHdfsUser() {
- return hdfsUser;
- }
-
- public void setHdfsUser(String hdfsUser) {
- this.hdfsUser = hdfsUser;
- }
-
public boolean isSecure() {
return secure;
}
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java
index 7465a50..282792a 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/conf/output/RolloverConfig.java
@@ -49,12 +49,32 @@ public class RolloverConfig {
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE,
description = "Rollover cloud log files after the log file size reach this limit",
- examples = {"1024KB"},
- defaultValue = "80MB",
+ examples = {"1024"},
+ defaultValue = "80",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
)
- @Value("${"+ LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE + ":80MB}")
- private String rolloverSize;
+ @Value("${"+ LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE + ":80}")
+ private Integer rolloverSize;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CLOUD_ROLLOVER_MAX_BACKUP_FILES,
+ description = "The number of max backup log files for rolled over logs.",
+ examples = {"50"},
+ defaultValue = "10",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.CLOUD_ROLLOVER_MAX_BACKUP_FILES + ":10}")
+ private Integer rolloverMaxBackupFiles;
+
+ @LogSearchPropertyDescription(
+ name = LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE_UNIT,
+ description = "Rollover cloud log file size unit (e.g: KB, MB etc.)",
+ examples = {"KB"},
+ defaultValue = "MB",
+ sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
+ )
+ @Value("${"+ LogFeederConstants.CLOUD_ROLLOVER_THRESHOLD_TIME_SIZE_UNIT + ":MB}")
+ private String rolloverSizeFormat;
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_ROLLOVER_USE_GZIP,
@@ -68,7 +88,7 @@ public class RolloverConfig {
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_ROLLOVER_IMMEDIATE_FLUSH,
- description = "Immediately flush cloud logs (to active location).",
+ description = "Immediately flush temporal cloud logs (to active location).",
examples = {"false"},
defaultValue = "true",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
@@ -78,7 +98,7 @@ public class RolloverConfig {
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_ROLLOVER_ON_SHUTDOWN,
- description = "Rollover log files on shutdown",
+ description = "Rollover temporal cloud log files on shutdown",
examples = {"false"},
defaultValue = "true",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
@@ -88,7 +108,7 @@ public class RolloverConfig {
@LogSearchPropertyDescription(
name = LogFeederConstants.CLOUD_ROLLOVER_ON_STARTUP,
- description = "Rollover log files on startup",
+ description = "Rollover temporal cloud log files on startup",
examples = {"false"},
defaultValue = "true",
sources = {LogFeederConstants.LOGFEEDER_PROPERTIES_FILE}
@@ -104,14 +124,30 @@ public class RolloverConfig {
this.rolloverThresholdTimeMins = rolloverThresholdTimeMins;
}
- public String getRolloverSize() {
+ public Integer getRolloverMaxBackupFiles() {
+ return rolloverMaxBackupFiles;
+ }
+
+ public void setRolloverMaxBackupFiles(Integer rolloverMaxBackupFiles) {
+ this.rolloverMaxBackupFiles = rolloverMaxBackupFiles;
+ }
+
+ public Integer getRolloverSize() {
return rolloverSize;
}
- public void setRolloverSize(String rolloverSize) {
+ public void setRolloverSize(Integer rolloverSize) {
this.rolloverSize = rolloverSize;
}
+ public String getRolloverSizeFormat() {
+ return rolloverSizeFormat;
+ }
+
+ public void setRolloverSizeFormat(String rolloverSizeFormat) {
+ this.rolloverSizeFormat = rolloverSizeFormat;
+ }
+
public boolean isUseGzip() {
return useGzip;
}
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
index f42e556..8201051 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageLoggerFactory.java
@@ -26,6 +26,7 @@ import org.apache.logging.log4j.Logger;
import org.apache.logging.log4j.core.LoggerContext;
import org.apache.logging.log4j.core.appender.RollingFileAppender;
import org.apache.logging.log4j.core.appender.rolling.CompositeTriggeringPolicy;
+import org.apache.logging.log4j.core.appender.rolling.DefaultRolloverStrategy;
import org.apache.logging.log4j.core.appender.rolling.OnStartupTriggeringPolicy;
import org.apache.logging.log4j.core.appender.rolling.SizeBasedTriggeringPolicy;
import org.apache.logging.log4j.core.config.AppenderRef;
@@ -64,8 +65,8 @@ public class CloudStorageLoggerFactory {
PatternLayout layout = PatternLayout.newBuilder()
.withPattern(PatternLayout.DEFAULT_CONVERSION_PATTERN).build();
- SizeBasedTriggeringPolicy sizeBasedTriggeringPolicy = SizeBasedTriggeringPolicy.createPolicy(
- logFeederProps.getRolloverConfig().getRolloverSize());
+ String rolloverSize = logFeederProps.getRolloverConfig().getRolloverSize().toString() + logFeederProps.getRolloverConfig().getRolloverSizeFormat();
+ SizeBasedTriggeringPolicy sizeBasedTriggeringPolicy = SizeBasedTriggeringPolicy.createPolicy(rolloverSize);
CustomTimeBasedTriggeringPolicy customTimeBasedTriggeringPolicy = CustomTimeBasedTriggeringPolicy
.createPolicy(String.valueOf(logFeederProps.getRolloverConfig().getRolloverThresholdTimeMins()));
@@ -80,6 +81,9 @@ public class CloudStorageLoggerFactory {
.createPolicy(sizeBasedTriggeringPolicy, customTimeBasedTriggeringPolicy);
}
+ DefaultRolloverStrategy defaultRolloverStrategy = DefaultRolloverStrategy.newBuilder().withMax(String.valueOf(
+ logFeederProps.getRolloverConfig().getRolloverMaxBackupFiles())).build();
+
boolean immediateFlush = logFeederProps.getRolloverConfig().isImmediateFlush();
RollingFileAppender appender = RollingFileAppender.newBuilder()
.withFileName(fileName)
@@ -87,6 +91,7 @@ public class CloudStorageLoggerFactory {
.withLayout(layout)
.withName(type)
.withPolicy(compositeTriggeringPolicy)
+ .withStrategy(defaultRolloverStrategy)
.withImmediateFlush(immediateFlush)
.build();
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
index ea52de5..b76f441 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/CloudStorageUploader.java
@@ -74,7 +74,7 @@ public class CloudStorageUploader extends Thread {
*/
void doUpload() {
try {
- final String archiveLogDir = String.join(File.separator, logFeederProps.getTmpDir(), uploaderType, "archived");
+ final String archiveLogDir = String.join(File.separator, logFeederProps.getRolloverConfig().getRolloverArchiveBaseDir(), uploaderType, "archived");
if (new File(archiveLogDir).exists()) {
String[] extensions = {"log", "gz"};
Collection<File> filesToUpload = FileUtils.listFiles(new File(archiveLogDir), extensions, true);
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/ExternalHDFSUploadClient.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/ExternalHDFSUploadClient.java
index cabf004..a23a715 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/ExternalHDFSUploadClient.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/ExternalHDFSUploadClient.java
@@ -47,9 +47,9 @@ public class ExternalHDFSUploadClient implements UploadClient {
@Override
public void init(LogFeederProps logFeederProps) {
logger.info("Initialize external HDFS client ...");
- if (StringUtils.isNotBlank(hdfsOutputConfig.getHdfsUser())) {
- logger.info("Using HADOOP_USER_NAME: {}", hdfsOutputConfig.getHdfsUser());
- System.setProperty("HADOOP_USER_NAME", hdfsOutputConfig.getHdfsUser());
+ if (StringUtils.isNotBlank(logFeederProps.getLogfeederHdfsUser())) {
+ logger.info("Using HADOOP_USER_NAME: {}", logFeederProps.getLogfeederHdfsUser());
+ System.setProperty("HADOOP_USER_NAME", logFeederProps.getLogfeederHdfsUser());
}
this.fs = LogFeederHDFSUtil.buildFileSystem(
hdfsOutputConfig.getHdfsHost(),
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
index 9e0a136..c2a8497 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/HDFSUploadClient.java
@@ -34,6 +34,8 @@ import java.io.IOException;
*/
public class HDFSUploadClient implements UploadClient {
+ private static final String FS_DEFAULT_FS = "fs.defaultFS";
+
private static final Logger logger = LogManager.getLogger(HDFSUploadClient.class);
private FileSystem fs;
@@ -43,7 +45,11 @@ public class HDFSUploadClient implements UploadClient {
logger.info("Initialize HDFS client (cloud mode), using core-site.xml from the classpath.");
Configuration configuration = new Configuration();
if (StringUtils.isNotBlank(logFeederProps.getCustomFs())) {
- configuration.set("fs.defaultFS", logFeederProps.getCustomFs());
+ configuration.set(FS_DEFAULT_FS, logFeederProps.getCustomFs());
+ }
+ if (StringUtils.isNotBlank(logFeederProps.getLogfeederHdfsUser()) && isHadoopFileSystem(configuration)) {
+ logger.info("Using HADOOP_USER_NAME: {}", logFeederProps.getLogfeederHdfsUser());
+ System.setProperty("HADOOP_USER_NAME", logFeederProps.getLogfeederHdfsUser());
}
this.fs = LogFeederHDFSUtil.buildFileSystem(configuration);
}
@@ -58,4 +64,8 @@ public class HDFSUploadClient implements UploadClient {
LogFeederHDFSUtil.closeFileSystem(fs);
}
+ private boolean isHadoopFileSystem(Configuration conf) {
+ return conf.get(FS_DEFAULT_FS).contains("hdfs://");
+ }
+
}
diff --git a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/UploadClientFactory.java b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/UploadClientFactory.java
index 2865b28..bea2943 100644
--- a/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/UploadClientFactory.java
+++ b/ambari-logsearch-logfeeder/src/main/java/org/apache/ambari/logfeeder/output/cloud/upload/UploadClientFactory.java
@@ -42,6 +42,7 @@ public class UploadClientFactory {
boolean useHdfsClient = logFeederProps.isUseCloudHdfsClient();
if (useHdfsClient && checkCoreSiteIsOnClasspath(logFeederProps)) {
logger.info("The core-site.xml from the classpath will be used to figure it out the cloud output settings.");
+ logFeederProps.setCloudStorageDestination(CloudStorageDestination.DEFAULT_FS);
return new HDFSUploadClient();
}
else if (CloudStorageDestination.HDFS.equals(destType)) {
diff --git a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
index daa9821..c7ea335 100644
--- a/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
+++ b/ambari-logsearch-logfeeder/src/main/resources/logfeeder.properties
@@ -53,13 +53,13 @@ logfeeder.cloud.storage.bucket.bootstrap=true
logfeeder.cloud.rollover.archive.base.dir=target/tmp
logfeeder.cloud.rollover.threshold.min=1000
-logfeeder.cloud.rollover.threshold.size=1K
+logfeeder.cloud.rollover.threshold.size=1
+logfeeder.cloud.rollover.threshold.size.unit=K
logfeeder.cloud.rollover.immediate.flush=true
logfeeder.hdfs.host=c7401.ambari.apache.org
logfeeder.hdfs.port=8020
logfeeder.hdfs.user=hdfs
-logfeeder.hdfs.output.base.dir=/apps/logfeeder
logfeeder.s3.endpoint=http://localhost:4569
logfeeder.s3.secret.key=MySecretKey