You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ambari.apache.org by mr...@apache.org on 2017/07/21 21:41:54 UTC
ambari git commit: AMBARI-21529: Integrate Registries and Mpack for
POST /mpacks using registry (mradhakrishnan)
Repository: ambari
Updated Branches:
refs/heads/branch-feature-AMBARI-14714 4450fd7ef -> 5465aaa56
AMBARI-21529: Integrate Registries and Mpack for POST /mpacks using registry (mradhakrishnan)
Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/5465aaa5
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/5465aaa5
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/5465aaa5
Branch: refs/heads/branch-feature-AMBARI-14714
Commit: 5465aaa56d7926206ba576911b2f5246231a79ca
Parents: 4450fd7
Author: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Authored: Fri Jul 21 14:40:52 2017 -0700
Committer: Madhuvanthi Radhakrishnan <mr...@hortonworks.com>
Committed: Fri Jul 21 14:40:52 2017 -0700
----------------------------------------------------------------------
.../internal/MpackResourceProvider.java | 33 +++++++--
.../ambari/server/mpack/MpackManager.java | 78 +++++++++++++-------
2 files changed, 81 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/ambari/blob/5465aaa5/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
index 1fd0343..e3afcde 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/controller/internal/MpackResourceProvider.java
@@ -18,6 +18,7 @@
package org.apache.ambari.server.controller.internal;
import java.io.IOException;
+import java.net.ConnectException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -28,6 +29,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.StaticallyInject;
import org.apache.ambari.server.api.services.parsers.BodyParseException;
import org.apache.ambari.server.controller.AmbariManagementController;
@@ -48,6 +50,9 @@ import org.apache.ambari.server.orm.dao.MpackDAO;
import org.apache.ambari.server.orm.dao.StackDAO;
import org.apache.ambari.server.orm.entities.MpackEntity;
import org.apache.ambari.server.orm.entities.StackEntity;
+import org.apache.ambari.server.registry.Registry;
+import org.apache.ambari.server.registry.RegistryMpack;
+import org.apache.ambari.server.registry.RegistryMpackVersion;
import org.apache.ambari.server.state.Packlet;
import com.google.inject.Inject;
@@ -121,9 +126,11 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider {
public RequestStatus createResourcesAuthorized(final Request request)
throws SystemException, UnsupportedPropertyException, ResourceAlreadyExistsException,
NoSuchParentResourceException, IllegalArgumentException {
+
+ MpackRequest mpackRequest = null;
Set<Resource> associatedResources = new HashSet<>();
try {
- MpackRequest mpackRequest = getRequest(request);
+ mpackRequest = getRequest(request);
if (mpackRequest == null)
throw new BodyParseException("Please provide " + MPACK_NAME + " ," + MPACK_VERSION + " ," + MPACK_URI);
MpackResponse response = getManagementController().registerMpack(mpackRequest);
@@ -140,6 +147,8 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider {
return getRequestStatus(null, associatedResources);
}
} catch (IOException e) {
+ if (e instanceof ConnectException)
+ throw new SystemException("The Mpack Uri : " + mpackRequest.getMpackUri() + " is not valid. Please try again");
e.printStackTrace();
} catch (BodyParseException e1) {
e1.printStackTrace();
@@ -147,18 +156,19 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider {
return null;
}
- public MpackRequest getRequest(Request request) {
+ public MpackRequest getRequest(Request request) throws AmbariException {
MpackRequest mpackRequest = new MpackRequest();
Set<Map<String, Object>> properties = request.getProperties();
for (Map propertyMap : properties) {
//Mpack Download url is either given in the request body or is fetched using the registry id
if (!propertyMap.containsKey(MPACK_URI) && !propertyMap.containsKey(REGISTRY_ID))
return null;
- //Fetch Mpack Download Url using the given registry id
+ //Fetch Mpack Download Url using the given registry id
else if (!propertyMap.containsKey(MPACK_URI)) {
- mpackRequest.setRegistryId((Long) propertyMap.get(REGISTRY_ID));
+ mpackRequest.setRegistryId(Long.valueOf ((String) propertyMap.get(REGISTRY_ID)));
mpackRequest.setMpackName((String) propertyMap.get(MPACK_NAME));
mpackRequest.setMpackVersion((String) propertyMap.get(MPACK_VERSION));
+ mpackRequest.setMpackUri(getMpackUri(mpackRequest));
}
//Directly download the mpack using the given url
else
@@ -168,6 +178,19 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider {
}
+ /***
+ * Uses the Registries functions to get the mpack uri.
+ * @param mpackRequest
+ * @return
+ * @throws AmbariException
+ */
+ private String getMpackUri(MpackRequest mpackRequest) throws AmbariException {
+ Registry registry = getManagementController().getRegistry(mpackRequest.getRegistryId());
+ RegistryMpack registryMpack = registry.getRegistryMpack(mpackRequest.getMpackName());
+ RegistryMpackVersion registryMpackVersion = registryMpack.getMpackVersion(mpackRequest.getMpackVersion());
+ return registryMpackVersion.getMpackUrl();
+ }
+
@Override
public Set<Resource> getResources(Request request, Predicate predicate)
throws SystemException, UnsupportedPropertyException,
@@ -216,7 +239,7 @@ public class MpackResourceProvider extends AbstractControllerResourceProvider {
if (propertyMap.containsKey(MPACK_ID)) {
Object objMpackId = propertyMap.get(MPACK_ID);
- if(objMpackId != null)
+ if (objMpackId != null)
mpackId = Long.valueOf((String) objMpackId);
MpackEntity entity = mpackDAO.findById(mpackId);
http://git-wip-us.apache.org/repos/asf/ambari/blob/5465aaa5/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
index 2bae72d..0c05292 100644
--- a/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
+++ b/ambari-server/src/main/java/org/apache/ambari/server/mpack/MpackManager.java
@@ -85,10 +85,12 @@ public class MpackManager {
* @throws IOException
*/
private void parseMpackDirectories() {
+
try {
for (final File dirEntry : mpacksStaging.listFiles()) {
if (dirEntry.isDirectory()) {
String mpackName = dirEntry.getName();
+
if (!mpackName.equals(MPACK_TAR_LOCATION)) {
for (final File file : dirEntry.listFiles()) {
if (file.isDirectory()) {
@@ -138,7 +140,6 @@ public class MpackManager {
mpackVersion = mpackRequest.getMpackVersion();
mpack.setRegistryId(mpackRequest.getRegistryId());
- //Todo : Madhu implement GET /registries/{registryId}/mpacks
mpackTarPath = downloadMpack(mpackRequest.getMpackUri());
if (createMpackDirectory(mpack, mpackTarPath)) {
@@ -156,10 +157,10 @@ public class MpackManager {
if (createMpackDirectory(mpack, mpackTarPath)) {
mpackDirectory = mpacksStaging + File.separator + mpack.getName() + File.separator + mpack.getVersion();
- mpack.setMpackUri(mpackRequest.getMpackUri());
}
}
extractMpackTar(mpack, mpackTarPath, mpackDirectory);
+ mpack.setMpackUri(mpackRequest.getMpackUri());
mpackId = populateDB(mpack);
if (mpackId != null) {
@@ -176,16 +177,17 @@ public class MpackManager {
/**
* Mpack is downloaded as a tar.gz file. It is extracted into mpack-v2-staging/{mpack-name}/{mpack-version}/ directory
*
- * @param mpack Mpack to process
- * @param mpackTarPath Path to mpack tarball
+ * @param mpack Mpack to process
+ * @param mpackTarPath Path to mpack tarball
* @param mpackDirectory Mpack directory
* @throws IOException
*/
private void extractMpackTar(Mpack mpack, Path mpackTarPath, String mpackDirectory) throws IOException {
+
TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(String.valueOf(mpackTarPath))))));
- // To read individual TAR file
TarArchiveEntry entry = null;
File outputFile = null;
+
//Create a loop to read every single entry in TAR file
while ((entry = mpackTarFile.getNextTarEntry()) != null) {
outputFile = new File(mpacksStaging, entry.getName());
@@ -204,7 +206,9 @@ public class MpackManager {
outputFileStream.close();
}
}
+
mpackTarFile.close();
+
String mpackTarDirectory = mpackTarPath.toString();
Path extractedMpackDirectory = Files.move
(Paths.get(mpacksStaging + File.separator + mpackTarDirectory.substring(mpackTarDirectory.lastIndexOf('/') + 1, mpackTarDirectory.indexOf(".tar")) + File.separator),
@@ -217,30 +221,30 @@ public class MpackManager {
* Reads the mpack.json file within the {mpack-name}.tar.gz file and populates Mpack object.
* Extract the mpack-name and mpack-version from mpack.json to create the new mpack directory to hold the mpack files.
*
- * @param mpack Mpack to process
+ * @param mpack Mpack to process
* @param mpackTarPath Path to mpack tarball
* @return boolean
* @throws IOException
*/
- private Boolean createMpackDirectory(Mpack mpack, Path mpackTarPath) throws IOException {
+ private Boolean createMpackDirectory(Mpack mpack, Path mpackTarPath) throws IOException, ResourceAlreadyExistsException {
+
TarArchiveInputStream mpackTarFile = new TarArchiveInputStream(new GzipCompressorInputStream(new BufferedInputStream(new FileInputStream(new File(mpackTarPath.toString())))));
- // To read individual TAR file
TarArchiveEntry entry = null;
String individualFiles;
int offset;
+
// Create a loop to read every single entry in TAR file
while ((entry = mpackTarFile.getNextTarEntry()) != null) {
// Get the name of the file
individualFiles = entry.getName();
String[] dirFile = individualFiles.split(File.separator);
+
//Search for mpack.json
String fileName = dirFile[dirFile.length - 1];
if (fileName.contains("mpack") && fileName.contains(".json")) {
- // Get Size of the file and create a byte array for the size
byte[] content = new byte[(int) entry.getSize()];
offset = 0;
LOG.debug("Size of the File is: " + entry.getSize());
- // Read file from the archive into byte array
mpackTarFile.read(content, offset, content.length - offset);
//Read the mpack.json file into Mpack Object for further use.
@@ -251,11 +255,20 @@ public class MpackManager {
mpackTarFile.close();
- File mpackDirectory = new File(mpacksStaging + File.separator + mpack.getName());
- if (!mpackDirectory.exists())
- return mpackDirectory.mkdir();
- else
- return true;
+ //Check if the mpack already exists
+ List<MpackEntity> mpackEntities = mpackDAO.findByNameVersion(mpack.getName(), mpack.getVersion());
+ if (mpackEntities.size() == 0) {
+ File mpackDirectory = new File(mpacksStaging + File.separator + mpack.getName());
+
+ if (!mpackDirectory.exists()) {
+ return mpackDirectory.mkdir();
+ } else {
+ return true;
+ }
+ } else {
+ String message = "Mpack :" + mpack.getName() + " version: " + mpack.getVersion() + " already exists in server";
+ throw new ResourceAlreadyExistsException(message);
+ }
}
}
@@ -270,18 +283,21 @@ public class MpackManager {
* @throws IOException
*/
private void createSymLinks(Mpack mpack) throws IOException {
+
String stackId = mpack.getStackId();
String[] stackMetaData = stackId.split("-");
String stackName = stackMetaData[0];
String stackVersion = stackMetaData[1];
File stack = new File(stackRoot + "/" + stackName);
- if(!stack.exists()) {
- stack.mkdir();
- }
Path stackPath = Paths.get(stackRoot + "/" + stackName + "/" + stackVersion);
Path mpackPath = Paths.get(mpacksStaging + "/" + mpack.getName() + "/" + mpack.getVersion());
- if(Files.isSymbolicLink(stackPath))
+
+ if (!stack.exists()) {
+ stack.mkdir();
+ }
+ if (Files.isSymbolicLink(stackPath)) {
Files.delete(stackPath);
+ }
Files.createSymbolicLink(stackPath, mpackPath);
}
@@ -292,15 +308,17 @@ public class MpackManager {
* @return
*/
public Path downloadMpack(String mpackURI) throws IOException {
+
URL url = new URL(mpackURI);
- String fileName = mpackURI.substring(mpackURI.lastIndexOf('/') + 1, mpackURI.length());
+ String mpackTarFile = mpackURI.substring(mpackURI.lastIndexOf('/') + 1, mpackURI.length());
File stagingDir = new File(mpacksStaging.toString() + File.separator + MPACK_TAR_LOCATION);
+ Path targetPath = new File(stagingDir.getPath() + File.separator + mpackTarFile).toPath();
+
if (!stagingDir.exists()) {
stagingDir.mkdir();
}
- Path targetPath = new File(stagingDir.getPath() + File.separator + fileName).toPath();
- Files.copy(url.openStream(), targetPath, StandardCopyOption.REPLACE_EXISTING);
+ Files.copy(url.openStream(), targetPath, StandardCopyOption.REPLACE_EXISTING);
return targetPath;
}
@@ -315,11 +333,14 @@ public class MpackManager {
* @return boolean
*/
protected boolean validateMpackInfo(String expectedMpackName, String expectedMpackVersion, String actualMpackName, String actualMpackVersion) {
- if (expectedMpackName.equalsIgnoreCase(actualMpackName) && expectedMpackVersion.equalsIgnoreCase(actualMpackVersion))
+
+ if (expectedMpackName.equalsIgnoreCase(actualMpackName) && expectedMpackVersion.equalsIgnoreCase(actualMpackVersion)) {
return true;
- else
+ }
+ else {
LOG.info("Incorrect information : Mismatch in - (" + expectedMpackName + "," + actualMpackName + ") or (" + expectedMpackVersion + "," + actualMpackVersion + ")");
- return false;
+ return false;
+ }
}
/**
@@ -330,15 +351,19 @@ public class MpackManager {
* @throws IOException
*/
protected Long populateDB(Mpack mpack) throws IOException {
+
String mpackName = mpack.getName();
String mpackVersion = mpack.getVersion();
List resultSet = mpackDAO.findByNameVersion(mpackName, mpackVersion);
+
if (resultSet.size() == 0) {
LOG.info("Adding mpack {}-{} to the database", mpackName, mpackVersion);
+
MpackEntity mpackEntity = new MpackEntity();
mpackEntity.setMpackName(mpackName);
mpackEntity.setMpackVersion(mpackVersion);
mpackEntity.setMpackUri(mpack.getMpackUri());
+ mpackEntity.setRegistryId(mpack.getRegistryId());
Long mpackId = mpackDAO.create(mpackEntity);
return mpackId;
@@ -363,12 +388,14 @@ public class MpackManager {
if (stackEntity == null) {
LOG.info("Adding stack {}-{} to the database", stackName, stackVersion);
stackEntity = new StackEntity();
+
stackEntity.setStackName(stackName);
stackEntity.setStackVersion(stackVersion);
stackEntity.setCurrentMpackId(mpack.getMpackId());
stackDAO.create(stackEntity);
} else {
LOG.info("Updating stack {}-{} to the database", stackName, stackVersion);
+
stackEntity.setCurrentMpackId(mpack.getMpackId());
stackDAO.merge(stackEntity);
}
@@ -380,6 +407,7 @@ public class MpackManager {
* @return ArrayList
*/
public ArrayList<Packlet> getPacklets(Long mpackId) {
+
Mpack mpack = mpackMap.get(mpackId);
if (mpack.getPacklets() != null)
return mpack.getPacklets();