You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by GitBox <gi...@apache.org> on 2018/11/21 11:07:06 UTC
[GitHub] oleewere closed pull request #17: AMBARI-24918 - Infra Manager: ssl
support
oleewere closed pull request #17: AMBARI-24918 - Infra Manager: ssl support
URL: https://github.com/apache/ambari-infra/pull/17
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java
index 7bc952ae..f149cd87 100644
--- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java
+++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/Solr.java
@@ -86,7 +86,7 @@ public void add(SolrInputDocument solrInputDocument) {
public void createSolrCollection(String collectionName) {
logger.info("Creating collection");
- runCommand(new String[]{"docker", "exec", "docker_solr_1", "solr", "create_collection", "-force", "-c", collectionName, "-d", Paths.get(configSetPath, "configsets", collectionName, "conf").toString(), "-n", collectionName + "_conf"});
+ runCommand(new String[]{"docker", "exec", "solr", "solr", "create_collection", "-force", "-c", collectionName, "-d", Paths.get(configSetPath, "configsets", collectionName, "conf").toString(), "-n", collectionName + "_conf"});
}
public QueryResponse query(SolrQuery query) {
diff --git a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java
index c400aeeb..6f174425 100644
--- a/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java
+++ b/ambari-infra-manager-it/src/test/java/org/apache/ambari/infra/solr/metrics/MetricsIT.java
@@ -52,7 +52,7 @@ public static void setupMetricsServer() throws Exception {
logger.info("Creating new docker containers for testing Ambari Infra Solr Metrics plugin ...");
runCommand(new String[]{shellScriptLocation, "start"});
- Solr solr = new Solr("/usr/lib/ambari-infra-solr/server/solr");
+ Solr solr = new Solr();
solr.waitUntilSolrIsUp();
solr.createSolrCollection(HADOOP_LOGS_COLLECTION);
diff --git a/ambari-infra-manager/docker/docker-compose.yml b/ambari-infra-manager/docker/docker-compose.yml
index 3fa21b2a..5051820e 100644
--- a/ambari-infra-manager/docker/docker-compose.yml
+++ b/ambari-infra-manager/docker/docker-compose.yml
@@ -15,6 +15,7 @@
version: '3.3'
services:
zookeeper:
+ container_name: zookeeper
image: zookeeper:${ZOOKEEPER_VERSION:-3.4.10}
restart: always
hostname: zookeeper
@@ -27,6 +28,7 @@ services:
ZOO_SERVERS: server.1=zookeeper:2888:3888
solr:
# TODO: use infra-solr
+ container_name: solr
image: solr:${SOLR_VERSION:-7.5.0}
restart: always
hostname: solr
@@ -47,6 +49,7 @@ services:
volumes:
- $AMBARI_INFRA_LOCATION/ambari-infra-manager/docker/configsets:/opt/solr/configsets
fakes3:
+ container_name: fakes3
image: localstack/localstack
hostname: fakes3
ports:
@@ -61,6 +64,7 @@ services:
env_file:
- Profile
namenode:
+ container_name: hdfs_namenode
image: flokkr/hadoop-hdfs-namenode:${HADOOP_VERSION:-3.0.0}
hostname: namenode
ports:
@@ -73,6 +77,7 @@ services:
networks:
- infra-network
datanode:
+ container_name: hdfs_datanode
image: flokkr/hadoop-hdfs-datanode:${HADOOP_VERSION:-3.0.0}
links:
- namenode
@@ -81,6 +86,7 @@ services:
networks:
- infra-network
inframanager:
+ container_name: inframanager
image: ambari-infra-manager:v1.0
restart: always
hostname: infra-manager.apache.org
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java
index 06174a0e..e560ae96 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/InfraManagerWebServerCustomizer.java
@@ -18,13 +18,17 @@
*/
package org.apache.ambari.infra.conf;
+import static org.apache.commons.lang3.StringUtils.isNotBlank;
+
import java.time.Duration;
import javax.inject.Inject;
+import org.apache.ambari.infra.conf.security.SslSecrets;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.autoconfigure.web.ServerProperties;
import org.springframework.boot.web.embedded.jetty.JettyServletWebServerFactory;
+import org.springframework.boot.web.server.Ssl;
import org.springframework.boot.web.server.WebServerFactoryCustomizer;
import org.springframework.stereotype.Component;
@@ -34,18 +38,47 @@
@Value("${infra-manager.server.port:61890}")
private int port;
+ @Value("${infra-manager.server.ssl.enabled:false}")
+ private boolean sslEnabled;
+
@Inject
private ServerProperties serverProperties;
+ @Inject
+ private SslSecrets sslSecrets;
+
private static final Integer SESSION_TIMEOUT = 60 * 30;
- private static final String INFRA_MANAGER_SESSIONID = "INFRAMANAGER_SESSIONID";
+ private static final String INFRA_MANAGER_SESSION_ID = "INFRAMANAGER_SESSIONID";
private static final String INFRA_MANAGER_APPLICATION_NAME = "infra-manager";
@Override
public void customize(JettyServletWebServerFactory factory) {
factory.setPort(port);
factory.setDisplayName(INFRA_MANAGER_APPLICATION_NAME);
- factory.getSession().getCookie().setName(INFRA_MANAGER_SESSIONID);
+ factory.getSession().getCookie().setName(INFRA_MANAGER_SESSION_ID);
factory.getSession().setTimeout(Duration.ofSeconds(SESSION_TIMEOUT));
+
+ Ssl ssl = new Ssl();
+ String keyStore = System.getProperty("javax.net.ssl.keyStore");
+ if (isNotBlank(keyStore)) {
+ ssl.setKeyStore(keyStore);
+ ssl.setKeyStoreType(System.getProperty("javax.net.ssl.keyStoreType"));
+ String keyStorePassword = sslSecrets.getKeyStorePassword().get().orElseThrow(() -> new IllegalStateException("Password for keystore is not set!"));
+ ssl.setKeyStorePassword(keyStorePassword);
+ System.setProperty("javax.net.ssl.keyStorePassword", keyStorePassword);
+ }
+
+ String trustStore = System.getProperty("javax.net.ssl.trustStore");
+ if (isNotBlank(trustStore)) {
+ ssl.setTrustStore(trustStore);
+ ssl.setTrustStoreType(System.getProperty("javax.net.ssl.trustStoreType"));
+ String trustStorePassword = sslSecrets.getTrustStorePassword().get().orElseThrow(() -> new IllegalStateException("Password for truststore is not set!"));
+ ssl.setKeyStorePassword(trustStorePassword);
+ System.setProperty("javax.net.ssl.trustStorePassword", trustStorePassword);
+ }
+
+ ssl.setEnabled(sslEnabled);
+
+ factory.setSsl(ssl);
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java
similarity index 70%
rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java
index 6d32963e..e8ab52e6 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositeSecret.java
@@ -20,17 +20,17 @@
import java.util.Optional;
-public class CompositePasswordStore implements PasswordStore {
- private PasswordStore[] passwordStores;
+public class CompositeSecret implements Secret {
+ private Secret[] secrets;
- public CompositePasswordStore(PasswordStore... passwordStores) {
- this.passwordStores = passwordStores;
+ public CompositeSecret(Secret... secrets) {
+ this.secrets = secrets;
}
@Override
- public Optional<String> getPassword(String propertyName) {
- for (PasswordStore passwordStore : passwordStores) {
- Optional<String> optionalPassword = passwordStore.getPassword(propertyName);
+ public Optional<String> get() {
+ for (Secret secret : secrets) {
+ Optional<String> optionalPassword = secret.get();
if (optionalPassword.isPresent())
return optionalPassword;
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java
similarity index 72%
rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java
index 8e3387b4..887767bb 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/EnvironmentalSecret.java
@@ -20,9 +20,16 @@
import java.util.Optional;
-public class SecurityEnvironment implements PasswordStore {
+public class EnvironmentalSecret implements Secret {
+
+ private final String environmentalVariableName;
+
+ public EnvironmentalSecret(String environmentalVariableName) {
+ this.environmentalVariableName = environmentalVariableName;
+ }
+
@Override
- public Optional<String> getPassword(String propertyName) {
- return Optional.ofNullable(System.getenv(propertyName));
+ public Optional<String> get() {
+ return Optional.ofNullable(System.getenv(environmentalVariableName));
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java
new file mode 100644
index 00000000..8fba08a4
--- /dev/null
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredential.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+import java.util.Optional;
+
+public class HadoopCredential implements Secret {
+
+ private final HadoopCredentialStore hadoopCredentialStore;
+ private final String propertyName;
+
+ public HadoopCredential(HadoopCredentialStore hadoopCredentialStore, String propertyName) {
+ this.propertyName = propertyName;
+ this.hadoopCredentialStore = hadoopCredentialStore;
+ }
+
+ @Override
+ public Optional<String> get() {
+ if (hadoopCredentialStore == null) {
+ return Optional.empty();
+ }
+
+ return hadoopCredentialStore.get(propertyName).map(String::new);
+ }
+}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
index 957a45d8..08a88047 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
@@ -21,13 +21,11 @@
import static org.apache.commons.lang.StringUtils.isBlank;
import static org.apache.commons.lang3.ArrayUtils.isNotEmpty;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.Optional;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-public class HadoopCredentialStore implements PasswordStore {
- private static final Logger logger = LogManager.getLogger(InfraManagerSecurityConfig.class);
+public class HadoopCredentialStore {
public static final String CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY = "hadoop.security.credential.provider.path";
private final String credentialStoreProviderPath;
@@ -36,8 +34,7 @@ public HadoopCredentialStore(String credentialStoreProviderPath) {
this.credentialStoreProviderPath = credentialStoreProviderPath;
}
- @Override
- public Optional<String> getPassword(String propertyName) {
+ public Optional<char[]> get(String key) {
try {
if (isBlank(credentialStoreProviderPath)) {
return Optional.empty();
@@ -45,11 +42,14 @@ public HadoopCredentialStore(String credentialStoreProviderPath) {
org.apache.hadoop.conf.Configuration config = new org.apache.hadoop.conf.Configuration();
config.set(CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY, credentialStoreProviderPath);
- char[] passwordChars = config.getPassword(propertyName);
- return (isNotEmpty(passwordChars)) ? Optional.of(new String(passwordChars)) : Optional.empty();
- } catch (Exception e) {
- logger.warn("Could not load password {} from credential store.", propertyName);
- return Optional.empty();
+ char[] passwordChars = config.getPassword(key);
+ return (isNotEmpty(passwordChars)) ? Optional.of(passwordChars) : Optional.empty();
+ } catch (IOException e) {
+ throw new UncheckedIOException(String.format("Could not load password %s from credential store.", key), e);
}
}
+
+ public Secret getSecret(String key) {
+ return new HadoopCredential(this, key);
+ }
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
index 45b79b36..0e5196d1 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
@@ -18,21 +18,44 @@
*/
package org.apache.ambari.infra.conf.security;
+import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY;
+
import org.springframework.beans.factory.annotation.Value;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
-import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY;
-
@Configuration
public class InfraManagerSecurityConfig {
@Value("${"+ CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY + ":}")
private String credentialStoreProviderPath;
+ @Bean
+ public HadoopCredentialStore hadoopCredentialStore() {
+ return new HadoopCredentialStore(credentialStoreProviderPath);
+ }
+
+ @Bean
+ public S3Secrets s3SecretStore(HadoopCredentialStore hadoopCredentialStore) {
+ return new S3Secrets(s3AccessKeyId(hadoopCredentialStore), s3SecretKeyId(hadoopCredentialStore));
+ }
+
+ private Secret s3AccessKeyId(HadoopCredentialStore hadoopCredentialStore) {
+ return new CompositeSecret(
+ hadoopCredentialStore.getSecret( "AWS_ACCESS_KEY_ID"),
+ new EnvironmentalSecret("AWS_ACCESS_KEY_ID"));
+ }
+
+ private Secret s3SecretKeyId(HadoopCredentialStore hadoopCredentialStore) {
+ return new CompositeSecret(
+ hadoopCredentialStore.getSecret( "AWS_SECRET_ACCESS_KEY"),
+ new EnvironmentalSecret("AWS_SECRET_ACCESS_KEY"));
+ }
@Bean
- public PasswordStore passwords() {
- return new CompositePasswordStore(new HadoopCredentialStore(credentialStoreProviderPath), new SecurityEnvironment());
+ public SslSecrets sslSecrets(HadoopCredentialStore hadoopCredentialStore) {
+ return new SslSecrets(
+ hadoopCredentialStore.getSecret("infra_manager_keystore_password"),
+ hadoopCredentialStore.getSecret("infra_manager_truststore_password"));
}
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java
new file mode 100644
index 00000000..30a1ca9c
--- /dev/null
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/S3Secrets.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+public class S3Secrets {
+ private final Secret s3AccessKeyId;
+ private final Secret s3SecretAccessKey;
+
+ public S3Secrets(Secret s3AccessKeyId, Secret s3SecretAccessKey) {
+ this.s3AccessKeyId = s3AccessKeyId;
+ this.s3SecretAccessKey = s3SecretAccessKey;
+ }
+
+
+ public Secret getS3AccessKeyId() {
+ return s3AccessKeyId;
+ }
+
+ public Secret getS3SecretAccessKey() {
+ return s3SecretAccessKey;
+ }
+}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java
similarity index 91%
rename from ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
rename to ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java
index 19848fea..e4f54a3e 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/Secret.java
@@ -20,6 +20,6 @@
import java.util.Optional;
-public interface PasswordStore {
- Optional<String> getPassword(String propertyName);
+public interface Secret {
+ Optional<String> get();
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java
new file mode 100644
index 00000000..6323e959
--- /dev/null
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SslSecrets.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+public class SslSecrets {
+ private final Secret keyStorePassword;
+ private final Secret trustStorePassword;
+
+ public SslSecrets(Secret keyStorePassword, Secret trustStorePassword) {
+ this.keyStorePassword = keyStorePassword;
+ this.trustStorePassword = trustStorePassword;
+ }
+
+ public Secret getKeyStorePassword() {
+ return keyStorePassword;
+ }
+
+ public Secret getTrustStorePassword() {
+ return trustStorePassword;
+ }
+}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
index 319cc5b1..b35c1d2d 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
@@ -27,12 +27,11 @@
import javax.inject.Inject;
import org.apache.ambari.infra.conf.InfraManagerDataConfig;
-import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.ambari.infra.conf.security.S3Secrets;
import org.apache.ambari.infra.job.AbstractJobsConfiguration;
import org.apache.ambari.infra.job.JobContextRepository;
import org.apache.ambari.infra.job.JobScheduler;
import org.apache.ambari.infra.job.ObjectSource;
-import org.apache.hadoop.fs.Path;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.springframework.batch.core.Job;
@@ -91,7 +90,7 @@ public DocumentExporter documentExporter(DocumentItemReader documentItemReader,
@Value("#{jobParameters[end]}") String intervalEnd,
DocumentWiper documentWiper,
JobContextRepository jobContextRepository,
- PasswordStore passwordStore) {
+ S3Secrets s3Secrets) {
File baseDir = new File(infraManagerDataConfig.getDataFolder(), "exporting");
CompositeFileAction fileAction = new CompositeFileAction(new BZip2Compressor());
@@ -99,7 +98,7 @@ public DocumentExporter documentExporter(DocumentItemReader documentItemReader,
case S3:
fileAction.add(new S3Uploader(
parameters.s3Properties().orElseThrow(() -> new IllegalStateException("S3 properties are not provided!")),
- passwordStore));
+ s3Secrets));
break;
case HDFS:
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
index 7c4de52a..3f541b8a 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
@@ -25,31 +25,40 @@
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Optional;
-import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.ambari.infra.conf.security.Secret;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-public class S3AccessCsv implements PasswordStore {
+public class S3AccessCsv implements Secret {
private static final Logger logger = LogManager.getLogger(S3AccessCsv.class);
+ public static final String ACCESS_KEY_ID = "Access key ID";
+ public static final String SECRET_ACCESS_KEY = "Secret access key";
- public static S3AccessCsv file(String path) {
+
+ public static S3AccessCsv file(String path, String propertyName) {
try {
- return new S3AccessCsv(new FileReader(path));
+ return new S3AccessCsv(new FileReader(path), propertyName);
} catch (FileNotFoundException e) {
throw new UncheckedIOException(e);
}
}
- private Map<String, String> passwordMap = new HashMap<>();
+ private final Reader reader;
+ private final String propertyName;
+
+ S3AccessCsv(Reader reader, String propertyName) {
+ this.reader = reader;
+ this.propertyName = propertyName;
+ }
- public S3AccessCsv(Reader reader) {
+ @Override
+ public Optional<String> get() {
try (CSVParser csvParser = CSVParser.parse(reader, DEFAULT.withHeader(
S3AccessKeyNames.AccessKeyId.getCsvName(), S3AccessKeyNames.SecretAccessKey.getCsvName()))) {
Iterator<CSVRecord> iterator = csvParser.iterator();
@@ -62,8 +71,8 @@ public S3AccessCsv(Reader reader) {
throw new S3AccessCsvFormatException("Csv file contains less than 2 columns!");
}
- checkColumnExists(record, S3AccessKeyNames.AccessKeyId);
- checkColumnExists(record, S3AccessKeyNames.SecretAccessKey);
+ checkColumnExists(record, ACCESS_KEY_ID);
+ checkColumnExists(record, SECRET_ACCESS_KEY);
if (!iterator.hasNext()) {
throw new S3AccessCsvFormatException("Csv file contains header only!");
@@ -72,23 +81,15 @@ public S3AccessCsv(Reader reader) {
record = iterator.next();
Map<String, Integer> header = csvParser.getHeaderMap();
- for (S3AccessKeyNames keyNames : S3AccessKeyNames.values())
- passwordMap.put(keyNames.getEnvVariableName(), record.get(header.get(keyNames.getCsvName())));
+ return Optional.ofNullable(record.get(header.get(propertyName)));
} catch (IOException e) {
throw new UncheckedIOException(e);
- } catch (S3AccessCsvFormatException e) {
- logger.warn("Unable to parse csv file: {}", e.getMessage());
}
}
- private void checkColumnExists(CSVRecord record, S3AccessKeyNames s3AccessKeyName) {
- if (!s3AccessKeyName.getCsvName().equals(record.get(s3AccessKeyName.getCsvName()))) {
- throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName.getCsvName()));
+ private void checkColumnExists(CSVRecord record, String s3AccessKeyName) {
+ if (!s3AccessKeyName.equals(record.get(s3AccessKeyName))) {
+ throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName));
}
}
-
- @Override
- public Optional<String> getPassword(String propertyName) {
- return Optional.ofNullable(passwordMap.get(propertyName));
- }
}
diff --git a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
index 3e1310a3..f3c92b7d 100644
--- a/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
+++ b/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
@@ -9,8 +9,9 @@
import java.security.InvalidKeyException;
import java.security.NoSuchAlgorithmException;
-import org.apache.ambari.infra.conf.security.CompositePasswordStore;
-import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.ambari.infra.conf.security.CompositeSecret;
+import org.apache.ambari.infra.conf.security.S3Secrets;
+import org.apache.ambari.infra.conf.security.Secret;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.xmlpull.v1.XmlPullParserException;
@@ -51,22 +52,28 @@
private final MinioClient client;
private final String keyPrefix;
private final String bucketName;
+ private final Secret s3AccessKey;
+ private final Secret s3SecretKey;
- public S3Uploader(S3Properties s3Properties, PasswordStore passwordStore) {
+ public S3Uploader(S3Properties s3Properties, S3Secrets s3Secrets) {
logger.info("Initializing S3 client with " + s3Properties);
this.keyPrefix = s3Properties.getS3KeyPrefix();
this.bucketName = s3Properties.getS3BucketName();
- PasswordStore compositePasswordStore = passwordStore;
- if (isNotBlank((s3Properties.getS3AccessFile())))
- compositePasswordStore = new CompositePasswordStore(passwordStore, S3AccessCsv.file(s3Properties.getS3AccessFile()));
+ if (isNotBlank(s3Properties.getS3AccessFile())) {
+ this.s3AccessKey = new CompositeSecret(s3Secrets.getS3AccessKeyId(), S3AccessCsv.file(s3Properties.getS3AccessFile(), "Access key ID"));
+ this.s3SecretKey = new CompositeSecret(s3Secrets.getS3SecretAccessKey(), S3AccessCsv.file(s3Properties.getS3AccessFile(), "Secret access key"));
+ }
+ else {
+ this.s3AccessKey = s3Secrets.getS3AccessKeyId();
+ this.s3SecretKey = s3Secrets.getS3SecretAccessKey();
+ }
try {
- client = new MinioClient(s3Properties.getS3EndPoint(), compositePasswordStore.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName())
- .orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")),
- compositePasswordStore.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName())
- .orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!")));
+ client = new MinioClient(s3Properties.getS3EndPoint(),
+ s3AccessKey.get().orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")),
+ s3SecretKey.get().orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!")));
if (!client.bucketExists(bucketName))
client.makeBucket(bucketName);
diff --git a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java
similarity index 70%
rename from ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
rename to ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java
index 26a6953d..78b0269c 100644
--- a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
+++ b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositeSecretTest.java
@@ -1,11 +1,11 @@
package org.apache.ambari.infra.conf.security;
-import org.junit.Test;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
import java.util.Optional;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -25,24 +25,24 @@
* specific language governing permissions and limitations
* under the License.
*/
-public class CompositePasswordStoreTest {
+public class CompositeSecretTest {
@Test
public void testGetPasswordReturnNullIfNoPasswordStoresWereAdded() {
- assertThat(new CompositePasswordStore().getPassword("any").isPresent(), is(false));
+ assertThat(new CompositeSecret().get().isPresent(), is(false));
}
@Test
public void testGetPasswordReturnNullIfPasswordNotFoundInAnyStore() {
- assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.empty()).getPassword("any").isPresent(), is(false));
+ assertThat(new CompositeSecret(Optional::empty, Optional::empty).get().isPresent(), is(false));
}
@Test
public void testGetPasswordReturnPasswordFromFirstStoreIfExists() {
- assertThat(new CompositePasswordStore((prop) -> Optional.of("Pass"), (prop) -> Optional.empty()).getPassword("any").get(), is("Pass"));
+ assertThat(new CompositeSecret(() -> Optional.of("Pass"), Optional::empty).get().get(), is("Pass"));
}
@Test
public void testGetPasswordReturnPasswordFromSecondStoreIfNotExistsInFirst() {
- assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.of("Pass")).getPassword("any").get(), is("Pass"));
+ assertThat(new CompositeSecret(Optional::empty, () -> Optional.of("Pass")).get().get(), is("Pass"));
}
}
\ No newline at end of file
diff --git a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
index e34a222c..eb4b011f 100644
--- a/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
+++ b/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
@@ -1,11 +1,12 @@
package org.apache.ambari.infra.job.archive;
-import org.junit.Test;
+import static org.apache.ambari.infra.job.archive.S3AccessCsv.ACCESS_KEY_ID;
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
import java.io.StringReader;
-import static org.hamcrest.core.Is.is;
-import static org.junit.Assert.assertThat;
+import org.junit.Test;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -33,38 +34,33 @@
private static final String ANY_CSV_FILE = "Column1,Column2\n" +
"Foo,Bar\n";
- @Test
+ @Test(expected = S3AccessCsvFormatException.class)
public void testGetPasswordReturnsNullIfInputIsEmpty() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(""));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(""), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
@Test
public void testGetPasswordReturnsAccessAndSecretKeyIfInputIsAValidS3AccessFile() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).get(), is("someKey"));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).get(), is("someSecret"));
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().get(), is("someKey"));
}
- @Test
- public void testGetPasswordReturnsNullIfNotAValidS3AccessFileProvided() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ @Test(expected = S3AccessCsvFormatException.class)
+ public void testGetPasswordThrowsExceptionIfNotAValidS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
- @Test
- public void testGetPasswordReturnsNullIfAHeaderOnlyS3AccessFileProvided() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n"));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ @Test(expected = S3AccessCsvFormatException.class)
+ public void testGetPasswordThrowsExceptionIfAHeaderOnlyS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n"), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
- @Test
- public void testGetPasswordReturnsNullIfOnlyOneValidColumnProvided() {
- S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n"));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
- assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ @Test(expected = S3AccessCsvFormatException.class)
+ public void testGetPasswordThrowsExceptionIfOnlyOneValidColumnProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n"), ACCESS_KEY_ID);
+ assertThat(accessCsv.get().isPresent(), is(false));
}
}
\ No newline at end of file
diff --git a/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh b/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
index 7ddb7579..a3df897d 100755
--- a/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
+++ b/ambari-infra-solr-plugin/docker/infra-solr-docker-compose.sh
@@ -101,8 +101,8 @@ function create_collection() {
pushd $sdir/../
local AMBARI_SOLR_MANAGER_LOCATION=$(pwd)
cd $AMBARI_SOLR_MANAGER_LOCATION/docker
- docker exec docker_solr_1 solr create_collection -force -c hadoop_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/hadoop_logs/conf -n hadoop_logs_conf
- docker exec docker_solr_1 solr create_collection -force -c audit_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/audit_logs/conf -n audit_logs_conf
+ docker exec solr solr create_collection -force -c hadoop_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/hadoop_logs/conf -n hadoop_logs_conf
+ docker exec solr solr create_collection -force -c audit_logs -d /usr/lib/ambari-infra-solr/server/solr/configsets/audit_logs/conf -n audit_logs_conf
popd
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services