You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2015/06/17 21:28:11 UTC
[3/6] metamodel git commit: Made HdfsResource non-closeable to avoid
statefulness.
Made HdfsResource non-closeable to avoid statefulness.
Integration test shows very similar performance metrics.
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/4621a28e
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/4621a28e
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/4621a28e
Branch: refs/heads/master
Commit: 4621a28e39383ac0509fe061efd849690e667f8f
Parents: 839926b
Author: Kasper Sørensen <i....@gmail.com>
Authored: Sat Jun 13 23:11:16 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Sat Jun 13 23:11:16 2015 +0200
----------------------------------------------------------------------
.../org/apache/metamodel/util/HdfsResource.java | 34 ++--------
.../util/HdfsResourceIntegrationTest.java | 66 +++++++++++++-------
.../apache/metamodel/util/HdfsResourceTest.java | 4 --
3 files changed, 50 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
index fcc5cc4..4590bde 100644
--- a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
+++ b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
@@ -18,7 +18,6 @@
*/
package org.apache.metamodel.util;
-import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -35,16 +34,13 @@ import org.apache.metamodel.MetaModelException;
* A {@link Resource} implementation that connects to Apache Hadoop's HDFS
* distributed file system.
*/
-public class HdfsResource implements Resource, Closeable {
+public class HdfsResource implements Resource {
private static final Pattern URL_PATTERN = Pattern.compile("hdfs://(.+):([0-9]+)/(.*)");
private final String _hostname;
private final int _port;
private final String _filepath;
-
- private FileSystem _fileSystem;
-
private Path _path;
/**
@@ -214,14 +210,11 @@ public class HdfsResource implements Resource, Closeable {
}
public FileSystem getHadoopFileSystem() {
- if (_fileSystem == null) {
- try {
- _fileSystem = FileSystem.get(getHadoopConfiguration());
- } catch (IOException e) {
- throw new MetaModelException("Could not connect to HDFS: " + e.getMessage(), e);
- }
+ try {
+ return FileSystem.get(getHadoopConfiguration());
+ } catch (IOException e) {
+ throw new MetaModelException("Could not connect to HDFS: " + e.getMessage(), e);
}
- return _fileSystem;
}
public Path getHadoopPath() {
@@ -232,23 +225,6 @@ public class HdfsResource implements Resource, Closeable {
}
@Override
- public void close() throws IOException {
- if (_fileSystem != null) {
- try {
- _fileSystem.close();
- } finally {
- _fileSystem = null;
- }
- }
- }
-
- @Override
- protected void finalize() throws Throwable {
- super.finalize();
- close();
- }
-
- @Override
public int hashCode() {
return Arrays.hashCode(new Object[] { _filepath, _hostname, _port });
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
index 4b80eeb..5ef5e53 100644
--- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
+++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
@@ -23,10 +23,18 @@ import java.io.FileReader;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Stopwatch;
import junit.framework.TestCase;
public class HdfsResourceIntegrationTest extends TestCase {
+
+ private static final Logger logger = LoggerFactory.getLogger(HdfsResourceIntegrationTest.class);
private boolean _configured;
private Properties _properties;
@@ -67,33 +75,49 @@ public class HdfsResourceIntegrationTest extends TestCase {
return;
}
final String contentString = "fun and games with Apache MetaModel and Hadoop is what we do";
+
+ final Stopwatch stopwatch = Stopwatch.createStarted();
final HdfsResource res1 = new HdfsResource(_hostname, _port, _filePath);
- try {
- assertFalse(res1.isExists());
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - start");
+
+ assertFalse(res1.isExists());
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - exists");
+
+ res1.write(new Action<OutputStream>() {
+ @Override
+ public void run(OutputStream out) throws Exception {
+ out.write(contentString.getBytes());
+ }
+ });
- res1.write(new Action<OutputStream>() {
- @Override
- public void run(OutputStream out) throws Exception {
- out.write(contentString.getBytes());
- }
- });
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - written");
- assertTrue(res1.isExists());
+ assertTrue(res1.isExists());
- final String str = res1.read(new Func<InputStream, String>() {
- @Override
- public String eval(InputStream in) {
- return FileHelper.readInputStreamAsString(in, "UTF8");
- }
- });
+ final String str1 = res1.read(new Func<InputStream, String>() {
+ @Override
+ public String eval(InputStream in) {
+ return FileHelper.readInputStreamAsString(in, "UTF8");
+ }
+ });
+ assertEquals(contentString, str1);
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read1");
- assertEquals(contentString, str);
- res1.getHadoopFileSystem().delete(res1.getHadoopPath(), false);
+ final String str2 = res1.read(new Func<InputStream, String>() {
+ @Override
+ public String eval(InputStream in) {
+ return FileHelper.readInputStreamAsString(in, "UTF8");
+ }
+ });
+ assertEquals(str1, str2);
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read2");
- assertFalse(res1.isExists());
+ res1.getHadoopFileSystem().delete(res1.getHadoopPath(), false);
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - deleted");
- } finally {
- res1.close();
- }
+ assertFalse(res1.isExists());
+
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - done");
+ stopwatch.stop();
}
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
index 7185e51..eace552 100644
--- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
+++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
@@ -38,9 +38,5 @@ public class HdfsResourceTest extends TestCase {
assertEquals("apache.txt", res3.getName());
assertFalse(res3.equals(res1));
-
- res1.close();
- res2.close();
- res3.close();
}
}