You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by ol...@apache.org on 2018/02/06 12:16:09 UTC
[ambari] branch trunk updated: [AMBARI-22886] Infra Manager: store
s3 credentials in Hadoop credential store (#261)
This is an automated email from the ASF dual-hosted git repository.
oleewere pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/ambari.git
The following commit(s) were added to refs/heads/trunk by this push:
new 9a3a920 [AMBARI-22886] Infra Manager: store s3 credentials in Hadoop credential store (#261)
9a3a920 is described below
commit 9a3a920e49bb0c21f083cef0eaaf2c1d5c597273
Author: kasakrisz <33...@users.noreply.github.com>
AuthorDate: Tue Feb 6 13:16:07 2018 +0100
[AMBARI-22886] Infra Manager: store s3 credentials in Hadoop credential store (#261)
* AMBARI-22886 Infra Manager: store s3 credentials in Hadoop credential store
* AMBARI-22886 Infra Manager: store s3 credentials in Hadoop credential store
- addendum: use Optional<String> as a return type of getPassword, remove unnecessary ArrayList
---
.../conf/security/CompositePasswordStore.java | 39 +++++++++
.../infra/conf/security/HadoopCredentialStore.java | 55 +++++++++++++
.../conf/security/InfraManagerSecurityConfig.java | 38 +++++++++
.../ambari/infra/conf/security/PasswordStore.java | 25 ++++++
.../infra/conf/security/SecurityEnvironment.java | 28 +++++++
.../infra/job/AbstractJobsConfiguration.java | 18 +++--
.../org/apache/ambari/infra/job/JobProperties.java | 2 +-
.../archive/DocumentArchivingConfiguration.java | 8 +-
.../job/archive/DocumentArchivingProperties.java | 54 ++-----------
.../ambari/infra/job/archive/S3AccessCsv.java | 94 ++++++++++++++++++++++
.../job/archive/S3AccessCsvFormatException.java | 25 ++++++
.../ambari/infra/job/archive/S3AccessKeyNames.java | 40 +++++++++
.../ambari/infra/job/archive/S3Properties.java | 28 ++-----
.../ambari/infra/job/archive/S3Uploader.java | 16 +++-
.../conf/security/CompositePasswordStoreTest.java | 48 +++++++++++
.../ambari/infra/job/archive/S3AccessCsvTest.java | 70 ++++++++++++++++
16 files changed, 510 insertions(+), 78 deletions(-)
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
new file mode 100644
index 0000000..6d32963
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/CompositePasswordStore.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+import java.util.Optional;
+
+public class CompositePasswordStore implements PasswordStore {
+ private PasswordStore[] passwordStores;
+
+ public CompositePasswordStore(PasswordStore... passwordStores) {
+ this.passwordStores = passwordStores;
+ }
+
+ @Override
+ public Optional<String> getPassword(String propertyName) {
+ for (PasswordStore passwordStore : passwordStores) {
+ Optional<String> optionalPassword = passwordStore.getPassword(propertyName);
+ if (optionalPassword.isPresent())
+ return optionalPassword;
+ }
+ return Optional.empty();
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
new file mode 100644
index 0000000..9e1a17f
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/HadoopCredentialStore.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Optional;
+
+import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.commons.lang3.ArrayUtils.isNotEmpty;
+
+public class HadoopCredentialStore implements PasswordStore {
+ private static final Logger LOG = LoggerFactory.getLogger(InfraManagerSecurityConfig.class);
+ public static final String CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY = "hadoop.security.credential.provider.path";
+
+ private final String credentialStoreProviderPath;
+
+ public HadoopCredentialStore(String credentialStoreProviderPath) {
+ this.credentialStoreProviderPath = credentialStoreProviderPath;
+ }
+
+ @Override
+ public Optional<String> getPassword(String propertyName) {
+ try {
+ if (isBlank(credentialStoreProviderPath)) {
+ return Optional.empty();
+ }
+
+ org.apache.hadoop.conf.Configuration config = new org.apache.hadoop.conf.Configuration();
+ config.set(CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY, credentialStoreProviderPath);
+ char[] passwordChars = config.getPassword(propertyName);
+ return (isNotEmpty(passwordChars)) ? Optional.of(new String(passwordChars)) : Optional.empty();
+ } catch (Exception e) {
+ LOG.warn("Could not load password {} from credential store.", propertyName);
+ return Optional.empty();
+ }
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
new file mode 100644
index 0000000..45b79b3
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/InfraManagerSecurityConfig.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+
+import static org.apache.ambari.infra.conf.security.HadoopCredentialStore.CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY;
+
+@Configuration
+public class InfraManagerSecurityConfig {
+
+ @Value("${"+ CREDENTIAL_STORE_PROVIDER_PATH_PROPERTY + ":}")
+ private String credentialStoreProviderPath;
+
+
+ @Bean
+ public PasswordStore passwords() {
+ return new CompositePasswordStore(new HadoopCredentialStore(credentialStoreProviderPath), new SecurityEnvironment());
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
new file mode 100644
index 0000000..19848fe
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/PasswordStore.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+import java.util.Optional;
+
+public interface PasswordStore {
+ Optional<String> getPassword(String propertyName);
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
new file mode 100644
index 0000000..8e3387b
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/conf/security/SecurityEnvironment.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.conf.security;
+
+import java.util.Optional;
+
+public class SecurityEnvironment implements PasswordStore {
+ @Override
+ public Optional<String> getPassword(String propertyName) {
+ return Optional.ofNullable(System.getenv(propertyName));
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/AbstractJobsConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/AbstractJobsConfiguration.java
index a57d0e0..02a6885 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/AbstractJobsConfiguration.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/AbstractJobsConfiguration.java
@@ -50,16 +50,20 @@ public abstract class AbstractJobsConfiguration<T extends JobProperties<T>> {
if (propertyMap == null)
return;
- for (String jobName : propertyMap.keySet())
- propertyMap.get(jobName).validate(jobName);
-
propertyMap.keySet().stream()
.filter(key -> propertyMap.get(key).isEnabled())
.forEach(jobName -> {
- LOG.info("Registering job {}", jobName);
- JobBuilder jobBuilder = jobs.get(jobName).listener(new JobsPropertyMap<>(propertyMap));
- Job job = buildJob(jobBuilder);
- jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName);
+ try {
+ propertyMap.get(jobName).validate(jobName);
+ LOG.info("Registering job {}", jobName);
+ JobBuilder jobBuilder = jobs.get(jobName).listener(new JobsPropertyMap<>(propertyMap));
+ Job job = buildJob(jobBuilder);
+ jobRegistryBeanPostProcessor.postProcessAfterInitialization(job, jobName);
+ }
+ catch (Exception e) {
+ LOG.warn("Unable to register job " + jobName, e);
+ propertyMap.get(jobName).setEnabled(false);
+ }
});
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java
index 53909ae..79406d0 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/JobProperties.java
@@ -68,7 +68,7 @@ public abstract class JobProperties<T extends JobProperties<T>> {
validate();
}
catch (Exception ex) {
- throw new JobConfigurationException(String.format("Configuration of job %s is invalid!", jobName), ex);
+ throw new JobConfigurationException(String.format("Configuration of job %s is invalid: %s!", jobName, ex.getMessage()), ex);
}
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
index 89f94bd..8358dd0 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingConfiguration.java
@@ -19,6 +19,7 @@
package org.apache.ambari.infra.job.archive;
import org.apache.ambari.infra.conf.InfraManagerDataConfig;
+import org.apache.ambari.infra.conf.security.PasswordStore;
import org.apache.ambari.infra.job.AbstractJobsConfiguration;
import org.apache.ambari.infra.job.JobContextRepository;
import org.apache.ambari.infra.job.JobScheduler;
@@ -86,13 +87,16 @@ public class DocumentArchivingConfiguration extends AbstractJobsConfiguration<Do
InfraManagerDataConfig infraManagerDataConfig,
@Value("#{jobParameters[end]}") String intervalEnd,
DocumentWiper documentWiper,
- JobContextRepository jobContextRepository) {
+ JobContextRepository jobContextRepository,
+ PasswordStore passwordStore) {
File baseDir = new File(infraManagerDataConfig.getDataFolder(), "exporting");
CompositeFileAction fileAction = new CompositeFileAction(new TarGzCompressor());
switch (properties.getDestination()) {
case S3:
- fileAction.add(new S3Uploader(properties.s3Properties().orElseThrow(() -> new IllegalStateException("S3 properties are not provided!"))));
+ fileAction.add(new S3Uploader(
+ properties.s3Properties().orElseThrow(() -> new IllegalStateException("S3 properties are not provided!")),
+ passwordStore));
break;
case HDFS:
org.apache.hadoop.conf.Configuration conf = new org.apache.hadoop.conf.Configuration();
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java
index f8fa33b..b26da36 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/DocumentArchivingProperties.java
@@ -19,23 +19,14 @@
package org.apache.ambari.infra.job.archive;
import org.apache.ambari.infra.job.JobProperties;
-import org.apache.commons.csv.CSVParser;
-import org.apache.commons.csv.CSVRecord;
import org.springframework.batch.core.JobParameters;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.UncheckedIOException;
-import java.util.Iterator;
-import java.util.Map;
import java.util.Optional;
-import java.util.function.Supplier;
import static java.util.Objects.requireNonNull;
import static org.apache.ambari.infra.job.archive.ExportDestination.HDFS;
import static org.apache.ambari.infra.job.archive.ExportDestination.LOCAL;
import static org.apache.ambari.infra.job.archive.ExportDestination.S3;
-import static org.apache.commons.csv.CSVFormat.DEFAULT;
import static org.apache.commons.lang.StringUtils.isBlank;
public class DocumentArchivingProperties extends JobProperties<DocumentArchivingProperties> {
@@ -50,47 +41,12 @@ public class DocumentArchivingProperties extends JobProperties<DocumentArchiving
private String s3KeyPrefix;
private String s3BucketName;
private String s3Endpoint;
- private transient Supplier<Optional<S3Properties>> s3Properties;
private String hdfsEndpoint;
private String hdfsDestinationDirectory;
public DocumentArchivingProperties() {
super(DocumentArchivingProperties.class);
- s3Properties = this::loadS3Properties;
- }
-
- private Optional<S3Properties> loadS3Properties() {
- if (isBlank(s3BucketName))
- return Optional.empty();
-
- String accessKey = System.getenv("AWS_ACCESS_KEY_ID");
- String secretKey = System.getenv("AWS_SECRET_ACCESS_KEY");
-
- if (isBlank(accessKey) || isBlank(secretKey)) {
- if (isBlank(s3AccessFile))
- return Optional.empty();
- try (CSVParser csvParser = CSVParser.parse(new FileReader(s3AccessFile), DEFAULT.withHeader("Access key ID", "Secret access key"))) {
- Iterator<CSVRecord> iterator = csvParser.iterator();
- if (!iterator.hasNext()) {
- return Optional.empty();
- }
-
- CSVRecord record = csvParser.iterator().next();
- Map<String, Integer> header = csvParser.getHeaderMap();
- accessKey = record.get(header.get("Access key ID"));
- secretKey = record.get(header.get("Secret access key"));
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- }
-
- return Optional.of(new S3Properties(
- accessKey,
- secretKey,
- s3KeyPrefix,
- s3BucketName,
- s3Endpoint));
}
public int getReadBlockSize() {
@@ -155,7 +111,6 @@ public class DocumentArchivingProperties extends JobProperties<DocumentArchiving
public void setS3AccessFile(String s3AccessFile) {
this.s3AccessFile = s3AccessFile;
- s3Properties = this::loadS3Properties;
}
public String getS3KeyPrefix() {
@@ -183,7 +138,14 @@ public class DocumentArchivingProperties extends JobProperties<DocumentArchiving
}
public Optional<S3Properties> s3Properties() {
- return s3Properties.get();
+ if (isBlank(s3BucketName))
+ return Optional.empty();
+
+ return Optional.of(new S3Properties(
+ s3AccessFile,
+ s3KeyPrefix,
+ s3BucketName,
+ s3Endpoint));
}
public String getHdfsEndpoint() {
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
new file mode 100644
index 0000000..879b58b
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsv.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.archive;
+
+import org.apache.ambari.infra.conf.security.PasswordStore;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.UncheckedIOException;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.apache.commons.csv.CSVFormat.DEFAULT;
+
+public class S3AccessCsv implements PasswordStore {
+ private static final Logger LOG = LoggerFactory.getLogger(S3AccessCsv.class);
+
+ public static S3AccessCsv file(String path) {
+ try {
+ return new S3AccessCsv(new FileReader(path));
+ } catch (FileNotFoundException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ private Map<String, String> passwordMap = new HashMap<>();
+
+ public S3AccessCsv(Reader reader) {
+ try (CSVParser csvParser = CSVParser.parse(reader, DEFAULT.withHeader(
+ S3AccessKeyNames.AccessKeyId.getCsvName(), S3AccessKeyNames.SecretAccessKey.getCsvName()))) {
+ Iterator<CSVRecord> iterator = csvParser.iterator();
+ if (!iterator.hasNext()) {
+ throw new S3AccessCsvFormatException("Csv file is empty!");
+ }
+
+ CSVRecord record = iterator.next();
+ if (record.size() < 2) {
+ throw new S3AccessCsvFormatException("Csv file contains less than 2 columns!");
+ }
+
+ checkColumnExists(record, S3AccessKeyNames.AccessKeyId);
+ checkColumnExists(record, S3AccessKeyNames.SecretAccessKey);
+
+ if (!iterator.hasNext()) {
+ throw new S3AccessCsvFormatException("Csv file contains header only!");
+ }
+
+ record = iterator.next();
+
+ Map<String, Integer> header = csvParser.getHeaderMap();
+ for (S3AccessKeyNames keyNames : S3AccessKeyNames.values())
+ passwordMap.put(keyNames.getEnvVariableName(), record.get(header.get(keyNames.getCsvName())));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ } catch (S3AccessCsvFormatException e) {
+ LOG.warn("Unable to parse csv file: {}", e.getMessage());
+ }
+ }
+
+ private void checkColumnExists(CSVRecord record, S3AccessKeyNames s3AccessKeyName) {
+ if (!s3AccessKeyName.getCsvName().equals(record.get(s3AccessKeyName.getCsvName()))) {
+ throw new S3AccessCsvFormatException(String.format("Csv file does not contain the required column: '%s'", s3AccessKeyName.getCsvName()));
+ }
+ }
+
+ @Override
+ public Optional<String> getPassword(String propertyName) {
+ return Optional.ofNullable(passwordMap.get(propertyName));
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsvFormatException.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsvFormatException.java
new file mode 100644
index 0000000..ef9d539
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessCsvFormatException.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.archive;
+
+public class S3AccessCsvFormatException extends RuntimeException {
+ public S3AccessCsvFormatException(String message) {
+ super(message);
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessKeyNames.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessKeyNames.java
new file mode 100644
index 0000000..e840d3b
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3AccessKeyNames.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.ambari.infra.job.archive;
+
+public enum S3AccessKeyNames {
+ AccessKeyId("AWS_ACCESS_KEY_ID", "Access key ID"),
+ SecretAccessKey("AWS_SECRET_ACCESS_KEY", "Secret access key");
+
+ private final String envVariableName;
+ private final String csvName;
+
+ S3AccessKeyNames(String envVariableName, String csvName) {
+ this.envVariableName = envVariableName;
+ this.csvName = csvName;
+ }
+
+ public String getEnvVariableName() {
+ return envVariableName;
+ }
+
+ public String getCsvName() {
+ return csvName;
+ }
+}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
index 88b71cf..59a4469 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Properties.java
@@ -21,28 +21,18 @@ package org.apache.ambari.infra.job.archive;
import static org.apache.commons.lang.StringUtils.isBlank;
public class S3Properties {
- private final String s3AccessKey;
- private final String s3SecretKey;
+ private final String s3AccessFile;
private final String s3KeyPrefix;
private final String s3BucketName;
private final String s3EndPoint;
- public S3Properties(String s3AccessKey, String s3SecretKey, String s3KeyPrefix, String s3BucketName, String s3EndPoint) {
- this.s3AccessKey = s3AccessKey;
- this.s3SecretKey = s3SecretKey;
+ public S3Properties(String s3AccessFile, String s3KeyPrefix, String s3BucketName, String s3EndPoint) {
+ this.s3AccessFile = s3AccessFile;
this.s3KeyPrefix = s3KeyPrefix;
this.s3BucketName = s3BucketName;
this.s3EndPoint = s3EndPoint;
}
- public String getS3AccessKey() {
- return s3AccessKey;
- }
-
- public String getS3SecretKey() {
- return s3SecretKey;
- }
-
public String getS3KeyPrefix() {
return s3KeyPrefix;
}
@@ -55,10 +45,14 @@ public class S3Properties {
return s3EndPoint;
}
+ public String getS3AccessFile() {
+ return s3AccessFile;
+ }
+
@Override
public String toString() {
return "S3Properties{" +
- "s3AccessKey='" + s3AccessKey + '\'' +
+ "s3AccessFile='" + s3AccessFile + '\'' +
", s3KeyPrefix='" + s3KeyPrefix + '\'' +
", s3BucketName='" + s3BucketName + '\'' +
", s3EndPoint='" + s3EndPoint + '\'' +
@@ -66,12 +60,6 @@ public class S3Properties {
}
public void validate() {
- if (isBlank(s3AccessKey))
- throw new IllegalArgumentException("The property s3AccessKey can not be null or empty string!");
-
- if (isBlank(s3SecretKey))
- throw new IllegalArgumentException("The property s3SecretKey can not be null or empty string!");
-
if (isBlank(s3BucketName))
throw new IllegalArgumentException("The property s3BucketName can not be null or empty string!");
}
diff --git a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
index 0ab68ed..2536cb5 100644
--- a/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
+++ b/ambari-infra/ambari-infra-manager/src/main/java/org/apache/ambari/infra/job/archive/S3Uploader.java
@@ -2,12 +2,15 @@ package org.apache.ambari.infra.job.archive;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
+import org.apache.ambari.infra.conf.security.CompositePasswordStore;
+import org.apache.ambari.infra.conf.security.PasswordStore;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import static org.apache.commons.lang.StringUtils.isBlank;
+import static org.apache.commons.lang.StringUtils.isNotBlank;
/*
* Licensed to the Apache Software Foundation (ASF) under one
@@ -35,12 +38,21 @@ public class S3Uploader extends AbstractFileAction {
private final String keyPrefix;
private final String bucketName;
- public S3Uploader(S3Properties s3Properties) {
+ public S3Uploader(S3Properties s3Properties, PasswordStore passwordStore) {
LOG.info("Initializing S3 client with " + s3Properties);
this.keyPrefix = s3Properties.getS3KeyPrefix();
this.bucketName = s3Properties.getS3BucketName();
- BasicAWSCredentials credentials = new BasicAWSCredentials(s3Properties.getS3AccessKey(), s3Properties.getS3SecretKey());
+
+ PasswordStore compositePasswordStore = passwordStore;
+ if (isNotBlank((s3Properties.getS3AccessFile())))
+ compositePasswordStore = new CompositePasswordStore(passwordStore, S3AccessCsv.file(s3Properties.getS3AccessFile()));
+
+ BasicAWSCredentials credentials = new BasicAWSCredentials(
+ compositePasswordStore.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName())
+ .orElseThrow(() -> new IllegalArgumentException("Access key Id is not present!")),
+ compositePasswordStore.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName())
+ .orElseThrow(() -> new IllegalArgumentException("Secret Access Key is not present!")));
client = new AmazonS3Client(credentials);
if (!isBlank(s3Properties.getS3EndPoint()))
client.setEndpoint(s3Properties.getS3EndPoint());
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
new file mode 100644
index 0000000..26a6953
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/conf/security/CompositePasswordStoreTest.java
@@ -0,0 +1,48 @@
+package org.apache.ambari.infra.conf.security;
+
+import org.junit.Test;
+
+import java.util.Optional;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class CompositePasswordStoreTest {
+ @Test
+ public void testGetPasswordReturnNullIfNoPasswordStoresWereAdded() {
+ assertThat(new CompositePasswordStore().getPassword("any").isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnNullIfPasswordNotFoundInAnyStore() {
+ assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.empty()).getPassword("any").isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnPasswordFromFirstStoreIfExists() {
+ assertThat(new CompositePasswordStore((prop) -> Optional.of("Pass"), (prop) -> Optional.empty()).getPassword("any").get(), is("Pass"));
+ }
+
+ @Test
+ public void testGetPasswordReturnPasswordFromSecondStoreIfNotExistsInFirst() {
+ assertThat(new CompositePasswordStore((prop) -> Optional.empty(), (prop) -> Optional.of("Pass")).getPassword("any").get(), is("Pass"));
+ }
+}
\ No newline at end of file
diff --git a/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
new file mode 100644
index 0000000..e34a222
--- /dev/null
+++ b/ambari-infra/ambari-infra-manager/src/test/java/org/apache/ambari/infra/job/archive/S3AccessCsvTest.java
@@ -0,0 +1,70 @@
+package org.apache.ambari.infra.job.archive;
+
+import org.junit.Test;
+
+import java.io.StringReader;
+
+import static org.hamcrest.core.Is.is;
+import static org.junit.Assert.assertThat;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+public class S3AccessCsvTest {
+
+ private static final String VALID_ACCESS_FILE = "Access key ID,Secret access key\n" +
+ "someKey,someSecret\n";
+
+ private static final String ANY_CSV_FILE = "Column1,Column2\n" +
+ "Foo,Bar\n";
+
+ @Test
+ public void testGetPasswordReturnsNullIfInputIsEmpty() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(""));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnsAccessAndSecretKeyIfInputIsAValidS3AccessFile() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(VALID_ACCESS_FILE));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).get(), is("someKey"));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).get(), is("someSecret"));
+ }
+
+ @Test
+ public void testGetPasswordReturnsNullIfNotAValidS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader(ANY_CSV_FILE));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnsNullIfAHeaderOnlyS3AccessFileProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Secret access key\n"));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+
+ @Test
+ public void testGetPasswordReturnsNullIfOnlyOneValidColumnProvided() {
+ S3AccessCsv accessCsv = new S3AccessCsv(new StringReader("Access key ID,Column\n"));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.AccessKeyId.getEnvVariableName()).isPresent(), is(false));
+ assertThat(accessCsv.getPassword(S3AccessKeyNames.SecretAccessKey.getEnvVariableName()).isPresent(), is(false));
+ }
+}
\ No newline at end of file
--
To stop receiving notification emails like this one, please contact
oleewere@apache.org.