You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mr...@apache.org on 2017/07/28 23:20:39 UTC
ambari git commit: AMBARI-21603: Minor fix to ensure mpack
depoloyment. (mradhakrishnan)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 294eddc42 -> e8c095402
AMBARI-21603: Minor fix to ensure mpack depoloyment. (mradhakrishnan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e8c09540
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e8c09540
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e8c09540
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: e8c09540287dc3b54ccdd25dc0fdd330262c99e4
Parents: 294eddc
Author: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Authored: Fri Jul 28 16:19:38 2017 -0700
Committer: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Committed: Fri Jul 28 16:19:38 2017 -0700
----------------------------------------------------------------------
.../ambari/server/mpack/MpackManager.java | 80 +++++++++++++++-----
.../apache/ambari/server/stack/StackModule.java | 2 +
.../src/test/resources/mpacks-v2/README.txt | 1 +
3 files changed, 63 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8c09540/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
index a02ea45..a8a227b 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
@@ -61,6 +61,8 @@ import com.google.inject.assistedinject.AssistedInject;
public class MpackManager {
private static final String MPACK_METADATA = "mpack.json";
private static final String MPACK_TAR_LOCATION = "staging";
+ private static final String SERVICES_DIRECTORY = "services";
+ private static final String PACKLETS_DIRECTORY = "packlets";
private final static Logger LOG = LoggerFactory.getLogger(MpackManager.class);
protected Map<Long, Mpack> mpackMap = new ConcurrentHashMap<>();
private File mpacksStaging;
@@ -97,13 +99,15 @@ public class MpackManager {
if (file.isDirectory()) {
String mpackVersion = file.getName();
List resultSet = mpackDAO.findByNameVersion(mpackName, mpackVersion);
- MpackEntity mpackEntity = (MpackEntity) resultSet.get(0);
-
- //Read the mpack.json file into Mpack Object for further use.
- String mpackJsonContents = new String((Files.readAllBytes(Paths.get(file + "/" + MPACK_METADATA))), "UTF-8");
- Gson gson = new Gson();
- Mpack existingMpack = gson.fromJson(mpackJsonContents, Mpack.class);
- mpackMap.put(mpackEntity.getMpackId(), existingMpack);
+ if (resultSet.size() > 0) {
+ MpackEntity mpackEntity = (MpackEntity) resultSet.get(0);
+
+ //Read the mpack.json file into Mpack Object for further use.
+ String mpackJsonContents = new String((Files.readAllBytes(Paths.get(file + "/" + MPACK_METADATA))), "UTF-8");
+ Gson gson = new Gson();
+ Mpack existingMpack = gson.fromJson(mpackJsonContents, Mpack.class);
+ mpackMap.put(mpackEntity.getMpackId(), existingMpack);
+ }
}
}
}
@@ -175,23 +179,19 @@ public class MpackManager {
}
}
- /**
- * Mpack is downloaded as a tar.gz file. It is extracted into mpack-v2-staging/{mpack-name}/{mpack-version}/ directory
- *
- * @param mpack Mpack to process
- * @param mpackTarPath Path to mpack tarball
- * @param mpackDirectory Mpack directory
+ /***
+ * A generic method to extract tar files.
+ * @param tarPath
* @throws IOException
*/
- private void extractMpackTar(Mpack mpack, Path mpackTarPath, String mpackDirectory) throws IOException {
-
- TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(String.valueOf(mpackTarPath))))));
+ private void extractTar(Path tarPath, File tempOutputDirectory) throws IOException{
+ TarArchiveInputStream tarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(String.valueOf(tarPath))))));
TarArchiveEntry entry = null;
File outputFile = null;
//Create a loop to read every single entry in TAR file
- while ((entry = mpackTarFile.getNextTarEntry()) != null) {
- outputFile = new File(mpacksStaging, entry.getName());
+ while ((entry = tarFile.getNextTarEntry()) != null) {
+ outputFile = new File(tempOutputDirectory, entry.getName());
if (entry.isDirectory()) {
LOG.debug("Attempting to write output directory" + outputFile.getAbsolutePath());
if (!outputFile.exists()) {
@@ -203,21 +203,61 @@ public class MpackManager {
} else {
LOG.debug("Creating output file %s." + outputFile.getAbsolutePath());
final OutputStream outputFileStream = new FileOutputStream(outputFile);
- IOUtils.copy(mpackTarFile, outputFileStream);
+ IOUtils.copy(tarFile, outputFileStream);
outputFileStream.close();
}
}
- mpackTarFile.close();
+ tarFile.close();
+ }
+
+ /**
+ * Mpack is downloaded as a tar.gz file. It is extracted into mpack-v2-staging/{mpack-name}/{mpack-version}/ directory
+ *
+ * @param mpack Mpack to process
+ * @param mpackTarPath Path to mpack tarball
+ * @param mpackDirectory Mpack directory
+ * @throws IOException
+ */
+ private void extractMpackTar(Mpack mpack, Path mpackTarPath, String mpackDirectory) throws IOException {
+
+ extractTar(mpackTarPath, mpacksStaging);
String mpackTarDirectory = mpackTarPath.toString();
Path extractedMpackDirectory = Files.move
(Paths.get(mpacksStaging + File.separator + mpackTarDirectory.substring(mpackTarDirectory.lastIndexOf('/') + 1, mpackTarDirectory.indexOf(".tar")) + File.separator),
Paths.get(mpackDirectory), StandardCopyOption.REPLACE_EXISTING);
+ createServicesDirectory(extractedMpackDirectory);
createSymLinks(mpack);
}
+ /***
+ * Create a services directory and extract all the services tar file inside it. This readies it for cluster deployment
+ * @param extractedMpackDirectory
+ * @throws IOException
+ */
+ private void createServicesDirectory(Path extractedMpackDirectory) throws IOException {
+ File servicesDir = new File(extractedMpackDirectory.toAbsolutePath() + File.separator + SERVICES_DIRECTORY);
+ if (!servicesDir.exists()) {
+ servicesDir.mkdir();
+ }
+ File packletDir = new File(extractedMpackDirectory.toAbsolutePath() + File.separator + PACKLETS_DIRECTORY);
+ for (final File serviceTar : packletDir.listFiles()) {
+ String serviceName = serviceTar.getName();
+ if (serviceName.contains("tar.gz")) {
+ extractTar(serviceTar.toPath(), packletDir);
+ String serviceTarDirectory = serviceTar.toString();
+ String serviceNameVersion = serviceTarDirectory.substring(serviceTarDirectory.lastIndexOf('/') + 1, serviceTarDirectory.indexOf(".tar"));
+ String serviceFolderName = serviceNameVersion.substring(0,serviceNameVersion.indexOf("-packlet")); //Can we assume the names of the tar files?
+ Path extractedServiceDirectory = Files.move
+ (Paths.get(packletDir + File.separator + serviceNameVersion),
+ Paths.get(servicesDir.toPath() + File.separator + serviceFolderName ), StandardCopyOption.REPLACE_EXISTING);
+ }
+ }
+ }
+
+
/**
* Reads the mpack.json file within the {mpack-name}.tar.gz file and populates Mpack object.
* Extract the mpack-name and mpack-version from mpack.json to create the new mpack directory to hold the mpack files.
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8c09540/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
index 7b2f87721..0d7ecf0 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/stack/StackModule.java
@@ -356,6 +356,8 @@ public class StackModule extends BaseModule<StackModule, StackInfo> implements V
String parent = serviceInfo.getParent();
if (parent != null) {
mergeServiceWithExplicitParent(service, parent, allStacks, commonServices, extensions);
+ }else {
+ serviceInfo.setServiceAdvisorType(ServiceInfo.ServiceAdvisorType.PYTHON);
}
}
}
http://git-wip-us.apache.org/repos/asf/ambari/blob/e8c09540/ambari-server/src/test/resources/mpacks-v2/README.txt
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/resources/mpacks-v2/README.txt b/ambari-server/src/test/resources/mpacks-v2/README.txt
new file mode 100644
index 0000000..00ca9bd
--- /dev/null
+++ b/ambari-server/src/test/resources/mpacks-v2/README.txt
@@ -0,0 +1 @@
+This is a place holder for unit tests which try to parse mpacks-v2 directory.
\ No newline at end of file