You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2019/04/03 21:52:11 UTC
[hadoop] branch trunk updated: HDDS-1358 : Recon Server REST API
not working as expected. (#668)
This is an automated email from the ASF dual-hosted git repository.
arp pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 8ff41d6 HDDS-1358 : Recon Server REST API not working as expected. (#668)
8ff41d6 is described below
commit 8ff41d62434266d49fab65abacea5f9a20fe8172
Author: avijayanhwx <14...@users.noreply.github.com>
AuthorDate: Wed Apr 3 14:52:06 2019 -0700
HDDS-1358 : Recon Server REST API not working as expected. (#668)
---
hadoop-ozone/ozone-recon/pom.xml | 69 ++++++++++-
....java => ReconGuiceServletContextListener.java} | 21 +++-
.../hadoop/ozone/recon/ReconRestServletModule.java | 134 +++++++++++++++++++++
.../org/apache/hadoop/ozone/recon/ReconServer.java | 12 +-
.../ozone/recon/api/ContainerKeyService.java | 50 ++++++--
.../hadoop/ozone/recon/api/types/KeyMetadata.java | 35 ++++++
.../recon/recovery/ReconOmMetadataManagerImpl.java | 2 +-
.../spi/impl/ContainerDBServiceProviderImpl.java | 2 +-
.../ozone/recon/tasks/ContainerKeyMapperTask.java | 9 +-
.../main/resources/webapps.recon.WEB-INF/web.xml | 25 ++--
.../ozone/recon/api/TestContainerKeyService.java | 22 +++-
hadoop-ozone/s3gateway/pom.xml | 16 +++
12 files changed, 356 insertions(+), 41 deletions(-)
diff --git a/hadoop-ozone/ozone-recon/pom.xml b/hadoop-ozone/ozone-recon/pom.xml
index 2ff57a5..ef23770 100644
--- a/hadoop-ozone/ozone-recon/pom.xml
+++ b/hadoop-ozone/ozone-recon/pom.xml
@@ -27,6 +27,20 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ozone-common</artifactId>
+ <exclusions>
+ <exclusion>
+ <artifactId>jersey-server</artifactId>
+ <groupId>com.sun.jersey</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jersey-core</artifactId>
+ <groupId>com.sun.jersey</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jersey-servlet</artifactId>
+ <groupId>com.sun.jersey</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -40,14 +54,38 @@
<dependency>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
- <version>4.1.0</version>
- <scope>compile</scope>
+ <version>${guice.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.containers</groupId>
+ <artifactId>jersey-container-servlet</artifactId>
+ <version>2.27</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-api</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.glassfish.jersey.containers</groupId>
<artifactId>jersey-container-servlet-core</artifactId>
<version>2.27</version>
- <scope>compile</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>guice-bridge</artifactId>
+ <version>2.5.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.core</groupId>
+ <artifactId>jersey-server</artifactId>
+ <version>2.27</version>
+ </dependency>
+ <dependency>
+ <groupId>org.glassfish.jersey.media</groupId>
+ <artifactId>jersey-media-json-jackson</artifactId>
+ <version>2.27</version>
</dependency>
<dependency>
<groupId>com.google.inject.extensions</groupId>
@@ -55,6 +93,25 @@
<version>4.1.0</version>
</dependency>
<dependency>
+ <groupId>org.glassfish.jersey.inject</groupId>
+ <artifactId>jersey-hk2</artifactId>
+ <version>2.27</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>hk2-api</artifactId>
+ <groupId>org.glassfish.hk2</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2.external</groupId>
+ <artifactId>aopalliance-repackaged</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-utils</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
@@ -70,6 +127,12 @@
<artifactId>powermock-module-junit4</artifactId>
<version>1.7.4</version>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconApplication.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconGuiceServletContextListener.java
similarity index 67%
rename from hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconApplication.java
rename to hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconGuiceServletContextListener.java
index 24ba5ee..ab11f0e 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconApplication.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconGuiceServletContextListener.java
@@ -15,15 +15,26 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.ozone.recon;
-import org.glassfish.jersey.server.ResourceConfig;
+import com.google.inject.Injector;
+import com.google.inject.servlet.GuiceServletContextListener;
/**
- * JaxRS resource definition.
+ * Servlet Context Listener that provides the Guice injector.
*/
-public class ReconApplication extends ResourceConfig {
- public ReconApplication() {
- packages("org.apache.hadoop.ozone.recon.api");
+public class ReconGuiceServletContextListener
+ extends GuiceServletContextListener {
+
+ private static Injector injector;
+
+ @Override
+ public Injector getInjector() {
+ return injector;
+ }
+
+ static void setInjector(Injector inj) {
+ injector = inj;
}
}
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java
new file mode 100644
index 0000000..5a69e66
--- /dev/null
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconRestServletModule.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.recon;
+
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.glassfish.hk2.api.ServiceLocator;
+import org.glassfish.jersey.internal.inject.InjectionManager;
+import org.glassfish.jersey.server.ResourceConfig;
+import org.glassfish.jersey.server.spi.Container;
+import org.glassfish.jersey.server.spi.ContainerLifecycleListener;
+import org.glassfish.jersey.servlet.ServletContainer;
+import org.jvnet.hk2.guice.bridge.api.GuiceBridge;
+import org.jvnet.hk2.guice.bridge.api.GuiceIntoHK2Bridge;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.inject.Injector;
+import com.google.inject.Scopes;
+import com.google.inject.servlet.ServletModule;
+
+/**
+ * Class to scan API Service classes and bind them to the injector.
+ */
+public abstract class ReconRestServletModule extends ServletModule {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ReconRestServletModule.class);
+
+ @Override
+ abstract protected void configureServlets();
+
+ /**
+ * Interface to provide packages for scanning.
+ */
+ public interface RestKeyBindingBuilder {
+ void packages(String... packages);
+ }
+
+ protected RestKeyBindingBuilder rest(String... urlPatterns) {
+ return new RestKeyBindingBuilderImpl(Arrays.asList(urlPatterns));
+ }
+
+ private class RestKeyBindingBuilderImpl implements RestKeyBindingBuilder {
+ private List<String> paths;
+
+ RestKeyBindingBuilderImpl(List<String> paths) {
+ this.paths = paths;
+ }
+
+ private void checkIfPackageExistsAndLog(String pkg) {
+ String resourcePath = pkg.replace(".", "/");
+ URL resource = getClass().getClassLoader().getResource(resourcePath);
+ if (resource != null) {
+ LOG.info("rest(" + paths + ").packages(" + pkg + ")");
+ } else {
+ LOG.info("No Beans in '" + pkg + "' found. Requests " + paths
+ + " will fail.");
+ }
+ }
+
+ @Override
+ public void packages(String... packages) {
+ StringBuilder sb = new StringBuilder();
+
+ for (String pkg : packages) {
+ if (sb.length() > 0) {
+ sb.append(',');
+ }
+ checkIfPackageExistsAndLog(pkg);
+ sb.append(pkg);
+ }
+ Map<String, String> params = new HashMap<>();
+ params.put("javax.ws.rs.Application",
+ GuiceResourceConfig.class.getCanonicalName());
+ if (sb.length() > 0) {
+ params.put("jersey.config.server.provider.packages", sb.toString());
+ }
+ bind(ServletContainer.class).in(Scopes.SINGLETON);
+ for (String path : paths) {
+ serve(path).with(ServletContainer.class, params);
+ }
+ }
+ }
+}
+
+/**
+ * Class to bridge Guice bindings to Jersey hk2 bindings.
+ */
+class GuiceResourceConfig extends ResourceConfig {
+ GuiceResourceConfig() {
+ register(new ContainerLifecycleListener() {
+ public void onStartup(Container container) {
+ ServletContainer servletContainer = (ServletContainer) container;
+ InjectionManager injectionManager = container.getApplicationHandler()
+ .getInjectionManager();
+ ServiceLocator serviceLocator = injectionManager
+ .getInstance(ServiceLocator.class);
+ GuiceBridge.getGuiceBridge().initializeGuiceBridge(serviceLocator);
+ GuiceIntoHK2Bridge guiceBridge = serviceLocator
+ .getService(GuiceIntoHK2Bridge.class);
+ Injector injector = (Injector) servletContainer.getServletContext()
+ .getAttribute(Injector.class.getName());
+ guiceBridge.bridgeGuiceInjector(injector);
+ }
+
+ public void onReload(Container container) {
+ }
+
+ public void onShutdown(Container container) {
+ }
+ });
+ }
+}
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
index 623e758..cea1681 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/ReconServer.java
@@ -62,7 +62,17 @@ public class ReconServer extends GenericCli {
OzoneConfiguration ozoneConfiguration = createOzoneConfiguration();
OzoneConfigurationProvider.setConfiguration(ozoneConfiguration);
- injector = Guice.createInjector(new ReconControllerModule());
+ injector = Guice.createInjector(new
+ ReconControllerModule(), new ReconRestServletModule() {
+ @Override
+ protected void configureServlets() {
+ rest("/api/*")
+ .packages("org.apache.hadoop.ozone.recon.api");
+ }
+ });
+
+ //Pass on injector to listener that does the Guice - Jersey HK2 bridging.
+ ReconGuiceServletContextListener.setInjector(injector);
httpServer = injector.getInstance(ReconHttpServer.class);
LOG.info("Starting Recon server");
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
index a62ad66..822c2ea 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/ContainerKeyService.java
@@ -19,32 +19,45 @@ package org.apache.hadoop.ozone.recon.api;
import java.io.IOException;
import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import java.util.stream.Collectors;
+import javax.inject.Inject;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.apache.hadoop.ozone.recon.ReconServer;
import org.apache.hadoop.ozone.recon.api.types.ContainerKeyPrefix;
import org.apache.hadoop.ozone.recon.api.types.KeyMetadata;
+import org.apache.hadoop.ozone.recon.api.types.KeyMetadata.ContainerBlockMetadata;
import org.apache.hadoop.ozone.recon.recovery.ReconOMMetadataManager;
import org.apache.hadoop.ozone.recon.spi.ContainerDBServiceProvider;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import com.google.inject.Inject;
/**
* Endpoint for querying keys that belong to a container.
*/
@Path("/containers")
+@Produces(MediaType.APPLICATION_JSON)
public class ContainerKeyService {
+ private static final Logger LOG = LoggerFactory.getLogger(ReconServer.class);
+
@Inject
private ContainerDBServiceProvider containerDBServiceProvider;
@@ -59,7 +72,7 @@ public class ContainerKeyService {
* @return {@link Response}
*/
@GET
- @Path("{id}")
+ @Path("/{id}")
public Response getKeysForContainer(@PathParam("id") Long containerId) {
Map<String, KeyMetadata> keyMetadataMap = new HashMap<>();
try {
@@ -80,19 +93,36 @@ public class ContainerKeyService {
}
// Filter keys by version.
- List<Long> matchedVersions = omKeyInfo.getKeyLocationVersions()
+ List<OmKeyLocationInfoGroup> matchedKeys = omKeyInfo
+ .getKeyLocationVersions()
.stream()
.filter(k -> (k.getVersion() == containerKeyPrefix.getKeyVersion()))
- .mapToLong(OmKeyLocationInfoGroup::getVersion)
- .boxed()
.collect(Collectors.toList());
+ List<ContainerBlockMetadata> blockIds = new ArrayList<>();
+ for (OmKeyLocationInfoGroup omKeyLocationInfoGroup : matchedKeys) {
+ List<OmKeyLocationInfo> omKeyLocationInfos = omKeyLocationInfoGroup
+ .getLocationList()
+ .stream()
+ .filter(c -> c.getContainerID() == containerId)
+ .collect(Collectors.toList());
+ for (OmKeyLocationInfo omKeyLocationInfo : omKeyLocationInfos) {
+ blockIds.add(new ContainerBlockMetadata(omKeyLocationInfo
+ .getContainerID(), omKeyLocationInfo.getLocalID()));
+ }
+ }
+
String ozoneKey = omMetadataManager.getOzoneKey(
omKeyInfo.getVolumeName(),
omKeyInfo.getBucketName(),
omKeyInfo.getKeyName());
if (keyMetadataMap.containsKey(ozoneKey)) {
- keyMetadataMap.get(ozoneKey).getVersions().addAll(matchedVersions);
+ keyMetadataMap.get(ozoneKey).getVersions()
+ .add(containerKeyPrefix.getKeyVersion());
+
+ keyMetadataMap.get(ozoneKey).getBlockIds().putAll(
+ Collections.singletonMap(containerKeyPrefix.getKeyVersion(),
+ blockIds));
} else {
KeyMetadata keyMetadata = new KeyMetadata();
keyMetadata.setBucket(omKeyInfo.getBucketName());
@@ -103,8 +133,14 @@ public class ContainerKeyService {
keyMetadata.setModificationTime(
Instant.ofEpochMilli(omKeyInfo.getModificationTime()));
keyMetadata.setDataSize(omKeyInfo.getDataSize());
- keyMetadata.setVersions(matchedVersions);
+ keyMetadata.setVersions(new ArrayList<Long>() {{
+ add(containerKeyPrefix.getKeyVersion());
+ }});
keyMetadataMap.put(ozoneKey, keyMetadata);
+ keyMetadata.setBlockIds(new TreeMap<Long,
+ List<ContainerBlockMetadata>>() {{
+ put(containerKeyPrefix.getKeyVersion(), blockIds);
+ }});
}
}
} catch (IOException ioEx) {
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyMetadata.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyMetadata.java
index 33ed285..3168263 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyMetadata.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/api/types/KeyMetadata.java
@@ -19,15 +19,18 @@ package org.apache.hadoop.ozone.recon.api.types;
import java.time.Instant;
import java.util.List;
+import java.util.Map;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.adapters.XmlJavaTypeAdapter;
/**
* Metadata object represents one key in the object store.
*/
+@XmlRootElement (name = "KeyMetadata")
@XmlAccessorType(XmlAccessType.FIELD)
public class KeyMetadata {
@@ -46,6 +49,9 @@ public class KeyMetadata {
@XmlElement(name = "Versions")
private List<Long> versions;
+ @XmlElement(name = "Blocks")
+ private Map<Long, List<ContainerBlockMetadata>> blockIds;
+
@XmlJavaTypeAdapter(IsoDateAdapter.class)
@XmlElement(name = "CreationTime")
private Instant creationTime;
@@ -109,4 +115,33 @@ public class KeyMetadata {
public void setVersions(List<Long> versions) {
this.versions = versions;
}
+
+ public Map<Long, List<ContainerBlockMetadata>> getBlockIds() {
+ return blockIds;
+ }
+
+ public void setBlockIds(Map<Long, List<ContainerBlockMetadata>> blockIds) {
+ this.blockIds = blockIds;
+ }
+
+ /**
+ * Class to hold ContainerID and BlockID.
+ */
+ public static class ContainerBlockMetadata {
+ private long containerID;
+ private long localID;
+
+ public ContainerBlockMetadata(long containerID, long localID) {
+ this.containerID = containerID;
+ this.localID = localID;
+ }
+
+ public long getContainerID() {
+ return containerID;
+ }
+
+ public long getLocalID() {
+ return localID;
+ }
+ }
}
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
index 145b95d..3b0fb49 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/recovery/ReconOmMetadataManagerImpl.java
@@ -88,7 +88,7 @@ public class ReconOmMetadataManagerImpl extends OmMetadataManagerImpl
if (oldDBLocation.exists()) {
LOG.info("Cleaning up old OM snapshot db at {}.",
oldDBLocation.getAbsolutePath());
- FileUtils.deleteQuietly(oldDBLocation);
+ FileUtils.deleteDirectory(oldDBLocation);
}
}
initializeNewRdbStore(newDbLocation);
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
index 351521d..ade35f0 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/spi/impl/ContainerDBServiceProviderImpl.java
@@ -88,7 +88,7 @@ public class ContainerDBServiceProviderImpl
if (oldDBLocation.exists()) {
LOG.info("Cleaning up old Recon Container DB at {}.",
oldDBLocation.getAbsolutePath());
- FileUtils.deleteQuietly(oldDBLocation);
+ FileUtils.deleteDirectory(oldDBLocation);
}
for (Map.Entry<ContainerKeyPrefix, Integer> entry :
containerKeyPrefixCounts.entrySet()) {
diff --git a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
index 66d8456..9ec1a79 100644
--- a/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
+++ b/hadoop-ozone/ozone-recon/src/main/java/org/apache/hadoop/ozone/recon/tasks/ContainerKeyMapperTask.java
@@ -85,8 +85,13 @@ public class ContainerKeyMapperTask implements Runnable {
long containerId = omKeyLocationInfo.getContainerID();
ContainerKeyPrefix containerKeyPrefix = new ContainerKeyPrefix(
containerId, key.toString(), keyVersion);
- containerDBServiceProvider.storeContainerKeyMapping(
- containerKeyPrefix, 1);
+ if (containerDBServiceProvider.getCountForForContainerKeyPrefix(
+ containerKeyPrefix) == 0) {
+ // Save on writes. No need to save same container-key prefix
+ // mapping again.
+ containerDBServiceProvider.storeContainerKeyMapping(
+ containerKeyPrefix, 1);
+ }
containerCount++;
}
}
diff --git a/hadoop-ozone/ozone-recon/src/main/resources/webapps.recon.WEB-INF/web.xml b/hadoop-ozone/ozone-recon/src/main/resources/webapps.recon.WEB-INF/web.xml
index c32f64c..972f3bb 100644
--- a/hadoop-ozone/ozone-recon/src/main/resources/webapps.recon.WEB-INF/web.xml
+++ b/hadoop-ozone/ozone-recon/src/main/resources/webapps.recon.WEB-INF/web.xml
@@ -14,20 +14,15 @@
<web-app version="3.0" xmlns="http://java.sun.com/xml/ns/javaee"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_3_0.xsd">
-
- <servlet>
- <servlet-name>jaxrs</servlet-name>
- <servlet-class>org.glassfish.jersey.servlet.ServletContainer</servlet-class>
- <init-param>
- <param-name>javax.ws.rs.Application</param-name>
- <param-value>org.apache.hadoop.ozone.recon.ReconApplication</param-value>
- </init-param>
- <load-on-startup>1</load-on-startup>
- </servlet>
- <servlet-mapping>
- <servlet-name>jaxrs</servlet-name>
+ <listener>
+ <listener-class>org.apache.hadoop.ozone.recon.ReconGuiceServletContextListener</listener-class>
+ </listener>
+ <filter>
+ <filter-name>guiceFilter</filter-name>
+ <filter-class>com.google.inject.servlet.GuiceFilter</filter-class>
+ </filter>
+ <filter-mapping>
+ <filter-name>guiceFilter</filter-name>
<url-pattern>/*</url-pattern>
- </servlet-mapping>
-
-
+ </filter-mapping>
</web-app>
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
index 58f3976..ad2d6a2 100644
--- a/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
+++ b/hadoop-ozone/ozone-recon/src/test/java/org/apache/hadoop/ozone/recon/api/TestContainerKeyService.java
@@ -31,6 +31,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
import javax.ws.rs.core.Response;
@@ -81,7 +82,7 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
private ReconOMMetadataManager reconOMMetadataManager;
private Injector injector;
private OzoneManagerServiceProviderImpl ozoneManagerServiceProvider;
- private ContainerKeyService containerKeyService = new ContainerKeyService();
+ private ContainerKeyService containerKeyService;
@Before
public void setUp() throws Exception {
@@ -98,11 +99,12 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
in(Singleton.class);
bind(ContainerDBServiceProvider.class).to(
ContainerDBServiceProviderImpl.class).in(Singleton.class);
- bind(ContainerKeyService.class).toInstance(containerKeyService);
ozoneManagerServiceProvider = new OzoneManagerServiceProviderImpl(
getTestOzoneConfiguration());
bind(OzoneManagerServiceProvider.class)
.toInstance(ozoneManagerServiceProvider);
+ containerKeyService = new ContainerKeyService();
+ bind(ContainerKeyService.class).toInstance(containerKeyService);
} catch (IOException e) {
Assert.fail();
}
@@ -119,12 +121,12 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
Pipeline pipeline = getRandomPipeline();
List<OmKeyLocationInfo> omKeyLocationInfoList = new ArrayList<>();
- BlockID blockID1 = new BlockID(1, 1);
+ BlockID blockID1 = new BlockID(1, 101);
OmKeyLocationInfo omKeyLocationInfo1 = getOmKeyLocationInfo(blockID1,
pipeline);
omKeyLocationInfoList.add(omKeyLocationInfo1);
- BlockID blockID2 = new BlockID(2, 1);
+ BlockID blockID2 = new BlockID(2, 102);
OmKeyLocationInfo omKeyLocationInfo2 = getOmKeyLocationInfo(blockID2,
pipeline);
omKeyLocationInfoList.add(omKeyLocationInfo2);
@@ -138,7 +140,7 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
Collections.singletonList(omKeyLocationInfoGroup));
List<OmKeyLocationInfoGroup> infoGroups = new ArrayList<>();
- BlockID blockID3 = new BlockID(1, 2);
+ BlockID blockID3 = new BlockID(1, 103);
OmKeyLocationInfo omKeyLocationInfo3 = getOmKeyLocationInfo(blockID3,
pipeline);
@@ -147,7 +149,7 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
infoGroups.add(new OmKeyLocationInfoGroup(0,
omKeyLocationInfoListNew));
- BlockID blockID4 = new BlockID(1, 3);
+ BlockID blockID4 = new BlockID(1, 104);
OmKeyLocationInfo omKeyLocationInfo4 = getOmKeyLocationInfo(blockID4,
pipeline);
@@ -186,12 +188,20 @@ public class TestContainerKeyService extends AbstractOMMetadataManagerTest {
KeyMetadata keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_one"));
assertTrue(keyMetadata.getVersions().size() == 1);
+ assertTrue(keyMetadata.getBlockIds().size() == 1);
+ Map<Long, List<KeyMetadata.ContainerBlockMetadata>> blockIds =
+ keyMetadata.getBlockIds();
+ assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 101);
keyMetadata = iterator.next();
assertTrue(keyMetadata.getKey().equals("key_two"));
assertTrue(keyMetadata.getVersions().size() == 2);
assertTrue(keyMetadata.getVersions().contains(0L) && keyMetadata
.getVersions().contains(1L));
+ assertTrue(keyMetadata.getBlockIds().size() == 2);
+ blockIds = keyMetadata.getBlockIds();
+ assertTrue(blockIds.get(0L).iterator().next().getLocalID() == 103);
+ assertTrue(blockIds.get(1L).iterator().next().getLocalID() == 104);
response = containerKeyService.getKeysForContainer(3L);
keyMetadataList = (Collection<KeyMetadata>) response.getEntity();
diff --git a/hadoop-ozone/s3gateway/pom.xml b/hadoop-ozone/s3gateway/pom.xml
index 9d95a19..3d47e72 100644
--- a/hadoop-ozone/s3gateway/pom.xml
+++ b/hadoop-ozone/s3gateway/pom.xml
@@ -55,6 +55,22 @@
<groupId>org.glassfish.jersey.inject</groupId>
<artifactId>jersey-hk2</artifactId>
<version>2.27</version>
+ <!-- The versions of these excluded dependencies are 2.5.0-beta. The
+ newer and release version 2.5.0 is being brought in by ozone-recon -->
+ <exclusions>
+ <exclusion>
+ <artifactId>hk2-api</artifactId>
+ <groupId>org.glassfish.hk2</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2.external</groupId>
+ <artifactId>aopalliance-repackaged</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.glassfish.hk2</groupId>
+ <artifactId>hk2-utils</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org