You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2019/02/24 21:55:05 UTC
[hadoop] branch trunk updated: HDDS-1089. Disable
OzoneFSStorageStatistics for hadoop versions older than 2.8. Contributed by
Elek, Marton.
This is an automated email from the ASF dual-hosted git repository.
aengineer pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 236b105 HDDS-1089. Disable OzoneFSStorageStatistics for hadoop versions older than 2.8. Contributed by Elek, Marton.
236b105 is described below
commit 236b105e415e4a10799b366a0d5a2d8b0b05a909
Author: Anu Engineer <ae...@apache.org>
AuthorDate: Sun Feb 24 13:53:38 2019 -0800
HDDS-1089. Disable OzoneFSStorageStatistics for hadoop versions older than 2.8.
Contributed by Elek, Marton.
---
.../hadoop/fs/ozone/OzoneClientAdapterFactory.java | 51 +++++++++++++++---
.../hadoop/fs/ozone/OzoneClientAdapterImpl.java | 54 ++++++++++++++++---
.../apache/hadoop/fs/ozone/OzoneFileSystem.java | 61 ++++++++++++++++------
3 files changed, 135 insertions(+), 31 deletions(-)
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java
index ca051dc..0352e7d 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterFactory.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs.ozone;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
@@ -42,8 +43,35 @@ public final class OzoneClientAdapterFactory {
@SuppressFBWarnings("DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED")
public static OzoneClientAdapter createAdapter(
String volumeStr,
- String bucketStr, OzoneFSStorageStatistics storageStatistics)
+ String bucketStr) throws IOException {
+ return createAdapter(volumeStr, bucketStr,
+ (aClass) -> (OzoneClientAdapter) aClass
+ .getConstructor(String.class, String.class)
+ .newInstance(
+ volumeStr,
+ bucketStr));
+ }
+
+
+ public static OzoneClientAdapter createAdapter(
+ String volumeStr,
+ String bucketStr,
+ OzoneFSStorageStatistics storageStatistics)
throws IOException {
+ return createAdapter(volumeStr, bucketStr,
+ (aClass) -> (OzoneClientAdapter) aClass
+ .getConstructor(String.class, String.class,
+ OzoneFSStorageStatistics.class)
+ .newInstance(
+ volumeStr,
+ bucketStr,
+ storageStatistics));
+ }
+
+ public static OzoneClientAdapter createAdapter(
+ String volumeStr,
+ String bucketStr,
+ OzoneClientAdapterCreator creator) throws IOException {
ClassLoader currentClassLoader = OzoneFileSystem.class.getClassLoader();
List<URL> urls = new ArrayList<>();
@@ -70,13 +98,10 @@ public final class OzoneClientAdapterFactory {
reflectionUtils.getMethod("getClassByName", String.class)
.invoke(null, "org.apache.ratis.grpc.GrpcFactory");
- OzoneClientAdapter ozoneClientAdapter = (OzoneClientAdapter) classLoader
- .loadClass("org.apache.hadoop.fs.ozone.OzoneClientAdapterImpl")
- .getConstructor(String.class, String.class,
- OzoneFSStorageStatistics.class)
- .newInstance(
- volumeStr,
- bucketStr, storageStatistics);
+ Class<?> aClass = classLoader
+ .loadClass("org.apache.hadoop.fs.ozone.OzoneClientAdapterImpl");
+ OzoneClientAdapter ozoneClientAdapter =
+ creator.createOzoneClientAdapter(aClass);
Thread.currentThread().setContextClassLoader(contextClassLoader);
@@ -119,4 +144,14 @@ public final class OzoneClientAdapterFactory {
}
+ /**
+ * Interface to create OzoneClientAdapter implementation with reflection.
+ */
+ @FunctionalInterface
+ interface OzoneClientAdapterCreator {
+ OzoneClientAdapter createOzoneClientAdapter(Class<?> clientAdapter)
+ throws NoSuchMethodException, IllegalAccessException,
+ InvocationTargetException, InstantiationException;
+ }
+
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
index a2de38e..8c69849 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapterImpl.java
@@ -54,11 +54,33 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
private ReplicationFactor replicationFactor;
private OzoneFSStorageStatistics storageStatistics;
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @param volumeStr Name of the volume to use.
+ * @param bucketStr Name of the bucket to use
+ * @param storageStatistics Storage statistic (optional, can be null)
+ * @throws IOException In case of a problem.
+ */
public OzoneClientAdapterImpl(String volumeStr, String bucketStr,
OzoneFSStorageStatistics storageStatistics) throws IOException {
this(createConf(), volumeStr, bucketStr, storageStatistics);
}
+ /**
+ * Create new OzoneClientAdapter implementation.
+ *
+ * @param volumeStr Name of the volume to use.
+ * @param bucketStr Name of the bucket to use
+ * @throws IOException In case of a problem.
+ */
+ public OzoneClientAdapterImpl(String volumeStr, String bucketStr)
+ throws IOException {
+ this(createConf(), volumeStr, bucketStr, null);
+ }
+
+
+
private static OzoneConfiguration createConf() {
ClassLoader contextClassLoader =
Thread.currentThread().getContextClassLoader();
@@ -102,13 +124,17 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
@Override
public InputStream createInputStream(String key) throws IOException {
- storageStatistics.incrementCounter(Statistic.OBJECTS_READ, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_READ, 1);
+ }
return bucket.readKey(key).getInputStream();
}
@Override
public OzoneFSOutputStream createKey(String key) throws IOException {
- storageStatistics.incrementCounter(Statistic.OBJECTS_CREATED, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_CREATED, 1);
+ }
OzoneOutputStream ozoneOutputStream =
bucket.createKey(key, 0, replicationType, replicationFactor,
new HashMap<>());
@@ -117,7 +143,9 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
@Override
public void renameKey(String key, String newKeyName) throws IOException {
- storageStatistics.incrementCounter(Statistic.OBJECTS_RENAMED, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_RENAMED, 1);
+ }
bucket.renameKey(key, newKeyName);
}
@@ -130,7 +158,9 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
@Override
public BasicKeyInfo getKeyInfo(String keyName) {
try {
- storageStatistics.incrementCounter(Statistic.OBJECTS_QUERY, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_QUERY, 1);
+ }
OzoneKey key = bucket.getKey(keyName);
return new BasicKeyInfo(
keyName,
@@ -167,7 +197,9 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
public boolean createDirectory(String keyName) {
try {
LOG.trace("creating dir for key:{}", keyName);
- storageStatistics.incrementCounter(Statistic.OBJECTS_CREATED, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_CREATED, 1);
+ }
bucket.createKey(keyName, 0, replicationType, replicationFactor,
new HashMap<>()).close();
return true;
@@ -187,7 +219,9 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
public boolean deleteObject(String keyName) {
LOG.trace("issuing delete for key" + keyName);
try {
- storageStatistics.incrementCounter(Statistic.OBJECTS_DELETED, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_DELETED, 1);
+ }
bucket.deleteKey(keyName);
return true;
} catch (IOException ioe) {
@@ -203,13 +237,17 @@ public class OzoneClientAdapterImpl implements OzoneClientAdapter {
@Override
public boolean hasNextKey(String key) {
- storageStatistics.incrementCounter(Statistic.OBJECTS_LIST, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_LIST, 1);
+ }
return bucket.listKeys(key).hasNext();
}
@Override
public Iterator<BasicKeyInfo> listKeys(String pathKey) {
- storageStatistics.incrementCounter(Statistic.OBJECTS_LIST, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.OBJECTS_LIST, 1);
+ }
return new IteratorAdapter(bucket.listKeys(pathKey));
}
diff --git a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
index 5337f2e..ad6de8a 100644
--- a/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
+++ b/hadoop-ozone/ozonefs/src/main/java/org/apache/hadoop/fs/ozone/OzoneFileSystem.java
@@ -125,14 +125,30 @@ public class OzoneFileSystem extends FileSystem {
boolean isolatedClassloader =
conf.getBoolean("ozone.fs.isolated-classloader", defaultValue);
- storageStatistics = (OzoneFSStorageStatistics)
- GlobalStorageStatistics.INSTANCE
- .put(OzoneFSStorageStatistics.NAME,
- OzoneFSStorageStatistics::new);
+ try {
+ //register only to the GlobalStorageStatistics if the class exists.
+ //This is required to support hadoop versions <2.7
+ Class.forName("org.apache.hadoop.fs.GlobalStorageStatistics");
+ storageStatistics = (OzoneFSStorageStatistics)
+ GlobalStorageStatistics.INSTANCE
+ .put(OzoneFSStorageStatistics.NAME,
+ OzoneFSStorageStatistics::new);
+ } catch (ClassNotFoundException e) {
+ //we don't support storage statistics for hadoop2.7 and older
+ }
+
if (isolatedClassloader) {
- this.adapter =
- OzoneClientAdapterFactory.createAdapter(volumeStr, bucketStr,
- storageStatistics);
+ try {
+ //register only to the GlobalStorageStatistics if the class exists.
+ //This is required to support hadoop versions <2.7
+ Class.forName("org.apache.hadoop.fs.GlobalStorageStatistics");
+ this.adapter =
+ OzoneClientAdapterFactory.createAdapter(volumeStr, bucketStr,
+ storageStatistics);
+ } catch (ClassNotFoundException e) {
+ this.adapter =
+ OzoneClientAdapterFactory.createAdapter(volumeStr, bucketStr);
+ }
} else {
OzoneConfiguration ozoneConfiguration;
if (conf instanceof OzoneConfiguration) {
@@ -188,7 +204,9 @@ public class OzoneFileSystem extends FileSystem {
@Override
public FSDataInputStream open(Path f, int bufferSize) throws IOException {
- storageStatistics.incrementCounter(Statistic.INVOCATION_OPEN, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.INVOCATION_OPEN, 1);
+ }
statistics.incrementWriteOps(1);
LOG.trace("open() path:{}", f);
final FileStatus fileStatus = getFileStatus(f);
@@ -207,7 +225,9 @@ public class OzoneFileSystem extends FileSystem {
short replication, long blockSize,
Progressable progress) throws IOException {
LOG.trace("create() path:{}", f);
- storageStatistics.incrementCounter(Statistic.INVOCATION_CREATE, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.INVOCATION_CREATE, 1);
+ }
statistics.incrementWriteOps(1);
final String key = pathToKey(f);
final FileStatus status;
@@ -240,8 +260,10 @@ public class OzoneFileSystem extends FileSystem {
short replication,
long blockSize,
Progressable progress) throws IOException {
- storageStatistics.incrementCounter(
- Statistic.INVOCATION_CREATE_NON_RECURSIVE, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(
+ Statistic.INVOCATION_CREATE_NON_RECURSIVE, 1);
+ }
statistics.incrementWriteOps(1);
final Path parent = path.getParent();
if (parent != null) {
@@ -296,7 +318,9 @@ public class OzoneFileSystem extends FileSystem {
*/
@Override
public boolean rename(Path src, Path dst) throws IOException {
- storageStatistics.incrementCounter(Statistic.INVOCATION_RENAME, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.INVOCATION_RENAME, 1);
+ }
statistics.incrementWriteOps(1);
if (src.equals(dst)) {
return true;
@@ -431,7 +455,9 @@ public class OzoneFileSystem extends FileSystem {
@Override
public boolean delete(Path f, boolean recursive) throws IOException {
- storageStatistics.incrementCounter(Statistic.INVOCATION_DELETE, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.INVOCATION_DELETE, 1);
+ }
statistics.incrementWriteOps(1);
LOG.debug("Delete path {} - recursive {}", f, recursive);
FileStatus status;
@@ -623,7 +649,9 @@ public class OzoneFileSystem extends FileSystem {
@Override
public FileStatus[] listStatus(Path f) throws IOException {
- storageStatistics.incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1);
+ if (storageStatistics != null) {
+ storageStatistics.incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1);
+ }
statistics.incrementReadOps(1);
LOG.trace("listStatus() path:{}", f);
ListStatusIterator iterator = new ListStatusIterator(f);
@@ -710,7 +738,10 @@ public class OzoneFileSystem extends FileSystem {
@Override
public FileStatus getFileStatus(Path f) throws IOException {
- storageStatistics.incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS, 1);
+ if (storageStatistics != null) {
+ storageStatistics
+ .incrementCounter(Statistic.INVOCATION_GET_FILE_STATUS, 1);
+ }
statistics.incrementReadOps(1);
LOG.trace("getFileStatus() path:{}", f);
Path qualifiedPath = f.makeQualified(uri, workingDir);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org