You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/11/29 18:13:17 UTC
[ambari-logsearch] branch master updated: AMBARI-24833. ABFS (ADLS
v2) fs implementation was not included on the classpath (#49)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ambari-logsearch.git
The following commit(s) were added to refs/heads/master by this push:
new e186a93 AMBARI-24833. ABFS (ADLS v2) fs implementation was not included on the classpath (#49)
e186a93 is described below
commit e186a93689cbb0d8c78e74227378c598eba94091
Author: Olivér Szabó <ol...@gmail.com>
AuthorDate: Thu Nov 29 19:13:13 2018 +0100
AMBARI-24833. ABFS (ADLS v2) fs implementation was not included on the classpath (#49)
* AMBARI-24833. ABFS (ADLS v2) fs implementation was not included on the classpath.
* Use common hadoop version
---
.../core-site-examples/abfs-core-site.xml | 54 ++++++++++++++++++++++
.../resources/core-site-examples/s3-core-site.xml | 30 ++++++++++++
.../core-site-examples/wasb-core-site.xml | 42 +++++++++++++++++
pom.xml | 2 +-
4 files changed, 127 insertions(+), 1 deletion(-)
diff --git a/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/abfs-core-site.xml b/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/abfs-core-site.xml
new file mode 100644
index 0000000..abbf6a3
--- /dev/null
+++ b/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/abfs-core-site.xml
@@ -0,0 +1,54 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>abfs://logsearch@myaccount.dfs.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.myaccount.dfs.core.windows.net</name>
+ <value>mySecretAccessKey</value>
+ </property>
+ <property>
+ <name>fs.adlsGen2.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem</value>
+ </property>
+ <property>
+ <name>fs.abfss.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem</value>
+ </property>
+ <property>
+ <name>fs.AbstractFileSystem.adlsGen2.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.Abfs</value>
+ </property>
+ <property>
+ <name>fs.AbstractFileSystem.abfss.impl</name>
+ <value>org.apache.hadoop.fs.azurebfs.Abfss</value>
+ </property>
+ <property>
+ <name>fs.azure.check.block.md5</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.azure.store.blob.md5</name>
+ <value>false</value>
+ </property>
+ <property>
+ <name>fs.azure.createRemoteFileSystemDuringInitialization</name>
+ <value>true</value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/s3-core-site.xml b/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/s3-core-site.xml
new file mode 100644
index 0000000..5807a54
--- /dev/null
+++ b/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/s3-core-site.xml
@@ -0,0 +1,30 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>s3a://logsearch</value>
+ </property>
+ <property>
+ <name>fs.s3a.access.key</name>
+ <value>MyAccessKey</value>
+ </property>
+ <property>
+ <name>fs.s3a.secret.key</name>
+ <value>MySecretKey</value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/wasb-core-site.xml b/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/wasb-core-site.xml
new file mode 100644
index 0000000..50f7bec
--- /dev/null
+++ b/ambari-logsearch-logfeeder/src/main/resources/core-site-examples/wasb-core-site.xml
@@ -0,0 +1,42 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>wasb://logsearch@myaccount.blob.core.windows.net</value>
+ </property>
+ <property>
+ <name>fs.azure.account.key.myaccount.blob.core.windows.net</name>
+ <value>mySecretAccessKey</value>
+ </property>
+ <property>
+ <name>fs.AbstractFileSystem.wasb.impl</name>
+ <value>org.apache.hadoop.fs.azure.Wasbs</value>
+ </property>
+ <property>
+ <name>fs.AbstractFileSystem.wasbs.impl</name>
+ <value>org.apache.hadoop.fs.azure.Wasbs</value>
+ </property>
+ <property>
+ <name>fs.azure.selfthrottling.read.factor</name>
+ <value>1.0</value>
+ </property>
+ <property>
+ <name>fs.azure.selfthrottling.write.factor</name>
+ <value>1.0</value>
+ </property>
+</configuration>
\ No newline at end of file
diff --git a/pom.xml b/pom.xml
index 91e5b09..53d99b5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,7 +84,7 @@
<deb.architecture>amd64</deb.architecture>
<deb.dependency.list>${deb.python.ver}</deb.dependency.list>
<solr.version>7.5.0</solr.version>
- <hadoop.version>3.1.1</hadoop.version>
+ <hadoop.version>3.1.1.3.0.2.0-50</hadoop.version>
<common.io.version>2.5</common.io.version>
<zookeeper.version>3.4.6.2.3.0.0-2557</zookeeper.version>
<forkCount>4</forkCount>