You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2015/06/17 21:28:09 UTC
[1/6] metamodel git commit: Added a proper integration test for the
HdfsResource
Repository: metamodel
Updated Branches:
refs/heads/master d43e907d9 -> 8156d76fd
Added a proper integration test for the HdfsResource
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/4496872e
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/4496872e
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/4496872e
Branch: refs/heads/master
Commit: 4496872efafd823cfc3331c1771f7fcdef8f66b7
Parents: bbc460f
Author: Kasper Sørensen <i....@gmail.com>
Authored: Sat Jun 13 20:01:22 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Sat Jun 13 20:01:22 2015 +0200
----------------------------------------------------------------------
...del-integrationtest-configuration.properties | 9 +++
.../util/HdfsResourceIntegrationTest.java | 81 ++++++++++++++++++++
2 files changed, 90 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4496872e/example-metamodel-integrationtest-configuration.properties
----------------------------------------------------------------------
diff --git a/example-metamodel-integrationtest-configuration.properties b/example-metamodel-integrationtest-configuration.properties
index ad556db..8f726fa 100644
--- a/example-metamodel-integrationtest-configuration.properties
+++ b/example-metamodel-integrationtest-configuration.properties
@@ -22,6 +22,15 @@
#mongodb.databaseName=metamodel_test
#mongodb.collectionName=my_collection
+# -------------------------
+# Hadoop module properties:
+# -------------------------
+
+#hadoop.hdfs.hostname=localhost
+#hadoop.hdfs.port=9000
+#hadoop.hdfs.file.path=/apache_metamodel_testfile.txt
+
+
# ------------------------
# HBase module properties:
# ------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4496872e/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
new file mode 100644
index 0000000..02a5dc8
--- /dev/null
+++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
@@ -0,0 +1,81 @@
+package org.apache.metamodel.util;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Properties;
+
+import junit.framework.TestCase;
+
+public class HdfsResourceIntegrationTest extends TestCase {
+
+ private boolean _configured;
+ private Properties _properties;
+ private String _filePath;
+ private String _hostname;
+ private int _port;
+
+ @Override
+ protected final void setUp() throws Exception {
+ super.setUp();
+
+ _properties = new Properties();
+ final File file = new File(getPropertyFilePath());
+ if (file.exists()) {
+ _properties.load(new FileReader(file));
+ _filePath = _properties.getProperty("hadoop.hdfs.file.path");
+ _hostname = _properties.getProperty("hadoop.hdfs.hostname");
+ final String portString = _properties.getProperty("hadoop.hdfs.port");
+ _configured = _filePath != null && _hostname != null && portString != null;
+ if (_configured) {
+ _port = Integer.parseInt(portString);
+ }
+ } else {
+ _configured = false;
+ }
+ }
+
+ private String getPropertyFilePath() {
+ String userHome = System.getProperty("user.home");
+ return userHome + "/metamodel-integrationtest-configuration.properties";
+ }
+
+ public void testReadOnRealHdfsInstall() throws Exception {
+ if (!_configured) {
+ System.err.println("!!! WARN !!! Hadoop HDFS integration test ignored\r\n"
+ + "Please configure Hadoop HDFS test-properties (" + getPropertyFilePath()
+ + "), to run integration tests");
+ return;
+ }
+ final String contentString = "fun and games with Apache MetaModel and Hadoop is what we do";
+ final HdfsResource res1 = new HdfsResource(_hostname, _port, _filePath);
+ try {
+ assertFalse(res1.isExists());
+
+ res1.write(new Action<OutputStream>() {
+ @Override
+ public void run(OutputStream out) throws Exception {
+ out.write(contentString.getBytes());
+ }
+ });
+
+ assertTrue(res1.isExists());
+
+ final String str = res1.read(new Func<InputStream, String>() {
+ @Override
+ public String eval(InputStream in) {
+ return FileHelper.readInputStreamAsString(in, "UTF8");
+ }
+ });
+
+ assertEquals(contentString, str);
+ res1.getHadoopFileSystem().delete(res1.getHadoopPath(), false);
+
+ assertFalse(res1.isExists());
+
+ } finally {
+ res1.close();
+ }
+ }
+}
[5/6] metamodel git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/metamodel
Posted by ka...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/metamodel
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/973a069d
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/973a069d
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/973a069d
Branch: refs/heads/master
Commit: 973a069d54f925a45c4db23e782901afd63e64bc
Parents: 08e3a8f d43e907
Author: Kasper Sørensen <i....@gmail.com>
Authored: Wed Jun 17 21:25:31 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Wed Jun 17 21:25:31 2015 +0200
----------------------------------------------------------------------
CHANGES.md | 1 +
.../cassandra/CassandraDataContext.java | 1 +
.../metamodel/cassandra/CassandraUtils.java | 2 +
.../cassandra/CassandraDataContextTest.java | 44 +++++++++++++++++---
4 files changed, 43 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[3/6] metamodel git commit: Made HdfsResource non-closeable to avoid
statefulness.
Posted by ka...@apache.org.
Made HdfsResource non-closeable to avoid statefulness.
Integration test shows very similar performance metrics.
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/4621a28e
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/4621a28e
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/4621a28e
Branch: refs/heads/master
Commit: 4621a28e39383ac0509fe061efd849690e667f8f
Parents: 839926b
Author: Kasper Sørensen <i....@gmail.com>
Authored: Sat Jun 13 23:11:16 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Sat Jun 13 23:11:16 2015 +0200
----------------------------------------------------------------------
.../org/apache/metamodel/util/HdfsResource.java | 34 ++--------
.../util/HdfsResourceIntegrationTest.java | 66 +++++++++++++-------
.../apache/metamodel/util/HdfsResourceTest.java | 4 --
3 files changed, 50 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
index fcc5cc4..4590bde 100644
--- a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
+++ b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
@@ -18,7 +18,6 @@
*/
package org.apache.metamodel.util;
-import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
@@ -35,16 +34,13 @@ import org.apache.metamodel.MetaModelException;
* A {@link Resource} implementation that connects to Apache Hadoop's HDFS
* distributed file system.
*/
-public class HdfsResource implements Resource, Closeable {
+public class HdfsResource implements Resource {
private static final Pattern URL_PATTERN = Pattern.compile("hdfs://(.+):([0-9]+)/(.*)");
private final String _hostname;
private final int _port;
private final String _filepath;
-
- private FileSystem _fileSystem;
-
private Path _path;
/**
@@ -214,14 +210,11 @@ public class HdfsResource implements Resource, Closeable {
}
public FileSystem getHadoopFileSystem() {
- if (_fileSystem == null) {
- try {
- _fileSystem = FileSystem.get(getHadoopConfiguration());
- } catch (IOException e) {
- throw new MetaModelException("Could not connect to HDFS: " + e.getMessage(), e);
- }
+ try {
+ return FileSystem.get(getHadoopConfiguration());
+ } catch (IOException e) {
+ throw new MetaModelException("Could not connect to HDFS: " + e.getMessage(), e);
}
- return _fileSystem;
}
public Path getHadoopPath() {
@@ -232,23 +225,6 @@ public class HdfsResource implements Resource, Closeable {
}
@Override
- public void close() throws IOException {
- if (_fileSystem != null) {
- try {
- _fileSystem.close();
- } finally {
- _fileSystem = null;
- }
- }
- }
-
- @Override
- protected void finalize() throws Throwable {
- super.finalize();
- close();
- }
-
- @Override
public int hashCode() {
return Arrays.hashCode(new Object[] { _filepath, _hostname, _port });
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
index 4b80eeb..5ef5e53 100644
--- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
+++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
@@ -23,10 +23,18 @@ import java.io.FileReader;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Stopwatch;
import junit.framework.TestCase;
public class HdfsResourceIntegrationTest extends TestCase {
+
+ private static final Logger logger = LoggerFactory.getLogger(HdfsResourceIntegrationTest.class);
private boolean _configured;
private Properties _properties;
@@ -67,33 +75,49 @@ public class HdfsResourceIntegrationTest extends TestCase {
return;
}
final String contentString = "fun and games with Apache MetaModel and Hadoop is what we do";
+
+ final Stopwatch stopwatch = Stopwatch.createStarted();
final HdfsResource res1 = new HdfsResource(_hostname, _port, _filePath);
- try {
- assertFalse(res1.isExists());
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - start");
+
+ assertFalse(res1.isExists());
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - exists");
+
+ res1.write(new Action<OutputStream>() {
+ @Override
+ public void run(OutputStream out) throws Exception {
+ out.write(contentString.getBytes());
+ }
+ });
- res1.write(new Action<OutputStream>() {
- @Override
- public void run(OutputStream out) throws Exception {
- out.write(contentString.getBytes());
- }
- });
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - written");
- assertTrue(res1.isExists());
+ assertTrue(res1.isExists());
- final String str = res1.read(new Func<InputStream, String>() {
- @Override
- public String eval(InputStream in) {
- return FileHelper.readInputStreamAsString(in, "UTF8");
- }
- });
+ final String str1 = res1.read(new Func<InputStream, String>() {
+ @Override
+ public String eval(InputStream in) {
+ return FileHelper.readInputStreamAsString(in, "UTF8");
+ }
+ });
+ assertEquals(contentString, str1);
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read1");
- assertEquals(contentString, str);
- res1.getHadoopFileSystem().delete(res1.getHadoopPath(), false);
+ final String str2 = res1.read(new Func<InputStream, String>() {
+ @Override
+ public String eval(InputStream in) {
+ return FileHelper.readInputStreamAsString(in, "UTF8");
+ }
+ });
+ assertEquals(str1, str2);
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - read2");
- assertFalse(res1.isExists());
+ res1.getHadoopFileSystem().delete(res1.getHadoopPath(), false);
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - deleted");
- } finally {
- res1.close();
- }
+ assertFalse(res1.isExists());
+
+ logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - done");
+ stopwatch.stop();
}
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/4621a28e/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
index 7185e51..eace552 100644
--- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
+++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceTest.java
@@ -38,9 +38,5 @@ public class HdfsResourceTest extends TestCase {
assertEquals("apache.txt", res3.getName());
assertFalse(res3.equals(res1));
-
- res1.close();
- res2.close();
- res3.close();
}
}
[4/6] metamodel git commit: Further improved the HdfsResource by
managing filesystem.close() better
Posted by ka...@apache.org.
Further improved the HdfsResource by managing filesystem.close() better
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/08e3a8f9
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/08e3a8f9
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/08e3a8f9
Branch: refs/heads/master
Commit: 08e3a8f9cda7b689f4ed0bafb9ad113276ca41d0
Parents: 4621a28
Author: Kasper Sørensen <i....@gmail.com>
Authored: Sun Jun 14 20:58:08 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Sun Jun 14 20:58:08 2015 +0200
----------------------------------------------------------------------
.../org/apache/metamodel/util/HdfsResource.java | 167 +++++++++++++------
.../util/HdfsResourceIntegrationTest.java | 3 -
hadoop/src/test/resources/log4j.xml | 21 +++
3 files changed, 135 insertions(+), 56 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/08e3a8f9/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
----------------------------------------------------------------------
diff --git a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
index 4590bde..c2f38ce 100644
--- a/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
+++ b/hadoop/src/main/java/org/apache/metamodel/util/HdfsResource.java
@@ -26,6 +26,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.metamodel.MetaModelException;
@@ -91,108 +92,168 @@ public class HdfsResource implements Resource {
@Override
public boolean isExists() {
- return doWithFileSystem(new UncheckedFunc<FileSystem, Boolean>() {
- @Override
- protected Boolean evalUnchecked(FileSystem fs) throws Exception {
- return fs.exists(getHadoopPath());
- }
- });
+ final FileSystem fs = getHadoopFileSystem();
+ try {
+ return fs.exists(getHadoopPath());
+ } catch (Exception e) {
+ throw wrapException(e);
+ } finally {
+ FileHelper.safeClose(fs);
+ }
}
@Override
public long getSize() {
- return doWithFileSystem(new UncheckedFunc<FileSystem, Long>() {
- @Override
- protected Long evalUnchecked(FileSystem fs) throws Exception {
- return fs.getFileStatus(getHadoopPath()).getLen();
- }
- });
+ final FileSystem fs = getHadoopFileSystem();
+ try {
+ return fs.getFileStatus(getHadoopPath()).getLen();
+ } catch (Exception e) {
+ throw wrapException(e);
+ } finally {
+ FileHelper.safeClose(fs);
+ }
}
@Override
public long getLastModified() {
- return doWithFileSystem(new UncheckedFunc<FileSystem, Long>() {
- @Override
- protected Long evalUnchecked(FileSystem fs) throws Exception {
- return fs.getFileStatus(getHadoopPath()).getModificationTime();
- }
- });
+ final FileSystem fs = getHadoopFileSystem();
+ try {
+ return fs.getFileStatus(getHadoopPath()).getModificationTime();
+ } catch (Exception e) {
+ throw wrapException(e);
+ } finally {
+ FileHelper.safeClose(fs);
+ }
}
@Override
public void write(final Action<OutputStream> writeCallback) throws ResourceException {
- final OutputStream out = doWithFileSystem(new UncheckedFunc<FileSystem, OutputStream>() {
- @Override
- protected OutputStream evalUnchecked(FileSystem fs) throws Exception {
- return fs.create(getHadoopPath(), true);
- }
- });
+ final FileSystem fs = getHadoopFileSystem();
try {
- writeCallback.run(out);
+ final FSDataOutputStream out = fs.create(getHadoopPath(), true);
+ try {
+ writeCallback.run(out);
+ } finally {
+ FileHelper.safeClose(out);
+ }
} catch (Exception e) {
throw wrapException(e);
} finally {
- FileHelper.safeClose(out);
+ FileHelper.safeClose(fs);
}
}
@Override
public void append(Action<OutputStream> appendCallback) throws ResourceException {
- final OutputStream out = doWithFileSystem(new UncheckedFunc<FileSystem, OutputStream>() {
- @Override
- protected OutputStream evalUnchecked(FileSystem fs) throws Exception {
- return fs.append(getHadoopPath());
- }
- });
+ final FileSystem fs = getHadoopFileSystem();
try {
- appendCallback.run(out);
+ final FSDataOutputStream out = fs.append(getHadoopPath());
+ try {
+ appendCallback.run(out);
+ } finally {
+ FileHelper.safeClose(out);
+ }
} catch (Exception e) {
throw wrapException(e);
} finally {
- FileHelper.safeClose(out);
+ FileHelper.safeClose(fs);
}
}
@Override
public InputStream read() throws ResourceException {
- return doWithFileSystem(new UncheckedFunc<FileSystem, InputStream>() {
+ final FileSystem fs = getHadoopFileSystem();
+ final InputStream in;
+ try {
+ in = fs.open(getHadoopPath());
+ } catch (Exception e) {
+ // we can close 'fs' in case of an exception
+ FileHelper.safeClose(fs);
+ throw wrapException(e);
+ }
+
+ // return a wrappper InputStream which manages the 'fs' closeable
+ return new InputStream() {
@Override
- protected InputStream evalUnchecked(FileSystem fs) throws Exception {
- return fs.open(getHadoopPath());
+ public int read() throws IOException {
+ return in.read();
}
- });
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ return in.read(b, off, len);
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ return in.read(b);
+ }
+
+ @Override
+ public boolean markSupported() {
+ return in.markSupported();
+ }
+
+ @Override
+ public synchronized void mark(int readlimit) {
+ in.mark(readlimit);
+ }
+
+ @Override
+ public int available() throws IOException {
+ return in.available();
+ }
+
+ @Override
+ public synchronized void reset() throws IOException {
+ in.reset();
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ return in.skip(n);
+ }
+
+ @Override
+ public void close() throws IOException {
+ super.close();
+ // need to close 'fs' when input stream is closed
+ FileHelper.safeClose(fs);
+ }
+ };
}
@Override
public void read(Action<InputStream> readCallback) throws ResourceException {
- final InputStream in = read();
+ final FileSystem fs = getHadoopFileSystem();
try {
- readCallback.run(in);
+ final InputStream in = fs.open(getHadoopPath());
+ try {
+ readCallback.run(in);
+ } finally {
+ FileHelper.safeClose(in);
+ }
} catch (Exception e) {
throw wrapException(e);
} finally {
- FileHelper.safeClose(in);
+ FileHelper.safeClose(fs);
}
}
@Override
public <E> E read(Func<InputStream, E> readCallback) throws ResourceException {
- final InputStream in = read();
+ final FileSystem fs = getHadoopFileSystem();
try {
- return readCallback.eval(in);
+ final InputStream in = fs.open(getHadoopPath());
+ try {
+ return readCallback.eval(in);
+ } finally {
+ FileHelper.safeClose(in);
+ }
} catch (Exception e) {
throw wrapException(e);
} finally {
- FileHelper.safeClose(in);
- }
- }
-
- private <E> E doWithFileSystem(Func<FileSystem, E> action) {
- final FileSystem hadoopFileSystem = getHadoopFileSystem();
- try {
- return action.eval(hadoopFileSystem);
- } catch (Exception e) {
- throw wrapException(e);
+ FileHelper.safeClose(fs);
}
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/08e3a8f9/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
index 5ef5e53..3f37379 100644
--- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
+++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
@@ -80,9 +80,6 @@ public class HdfsResourceIntegrationTest extends TestCase {
final HdfsResource res1 = new HdfsResource(_hostname, _port, _filePath);
logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - start");
- assertFalse(res1.isExists());
- logger.info(stopwatch.elapsed(TimeUnit.MILLISECONDS) + " - exists");
-
res1.write(new Action<OutputStream>() {
@Override
public void run(OutputStream out) throws Exception {
http://git-wip-us.apache.org/repos/asf/metamodel/blob/08e3a8f9/hadoop/src/test/resources/log4j.xml
----------------------------------------------------------------------
diff --git a/hadoop/src/test/resources/log4j.xml b/hadoop/src/test/resources/log4j.xml
new file mode 100644
index 0000000..29f497f
--- /dev/null
+++ b/hadoop/src/test/resources/log4j.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!DOCTYPE log4j:configuration SYSTEM "log4j.dtd">
+<log4j:configuration xmlns:log4j="http://jakarta.apache.org/log4j/">
+
+ <appender name="consoleAppender" class="org.apache.log4j.ConsoleAppender">
+ <param name="Target" value="System.out" />
+ <layout class="org.apache.log4j.PatternLayout">
+ <param name="ConversionPattern" value="%-5p %d{HH:mm:ss} %c{1} - %m%n" />
+ </layout>
+ </appender>
+
+ <logger name="org.apache.metamodel">
+ <level value="info" />
+ </logger>
+
+ <root>
+ <priority value="fatal" />
+ <appender-ref ref="consoleAppender" />
+ </root>
+
+</log4j:configuration>
\ No newline at end of file
[2/6] metamodel git commit: Added license header
Posted by ka...@apache.org.
Added license header
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/839926bc
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/839926bc
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/839926bc
Branch: refs/heads/master
Commit: 839926bc66eb0988cfb0764c511dd088d886517a
Parents: 4496872
Author: Kasper Sørensen <i....@gmail.com>
Authored: Sat Jun 13 20:01:46 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Sat Jun 13 20:01:46 2015 +0200
----------------------------------------------------------------------
.../util/HdfsResourceIntegrationTest.java | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/839926bc/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
----------------------------------------------------------------------
diff --git a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
index 02a5dc8..4b80eeb 100644
--- a/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
+++ b/hadoop/src/test/java/org/apache/metamodel/util/HdfsResourceIntegrationTest.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
package org.apache.metamodel.util;
import java.io.File;
[6/6] metamodel git commit: METAMODEL-151 fixed. Fixes #28
Posted by ka...@apache.org.
METAMODEL-151 fixed.
Fixes #28
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/8156d76f
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/8156d76f
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/8156d76f
Branch: refs/heads/master
Commit: 8156d76fd10a9ccc3ae88563e417b9c8700f22a3
Parents: 973a069
Author: Kasper Sørensen <i....@gmail.com>
Authored: Wed Jun 17 21:27:06 2015 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Wed Jun 17 21:27:06 2015 +0200
----------------------------------------------------------------------
CHANGES.md | 1 +
.../jdbc/dialects/PostgresqlQueryRewriter.java | 3 +
.../jdbc/integrationtests/PostgresqlTest.java | 60 ++++++++++++++++++++
3 files changed, 64 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/8156d76f/CHANGES.md
----------------------------------------------------------------------
diff --git a/CHANGES.md b/CHANGES.md
index 6991fe3..c6c6f80 100644
--- a/CHANGES.md
+++ b/CHANGES.md
@@ -2,6 +2,7 @@
* [METAMODEL-148] - Added a 'hadoop' module with a HdfsResource class to allow CSV, Excel and Fixed-width file access on HDFS.
* [METAMODEL-149] - Added support for COUNTER data type in Cassandra.
+ * [METAMODEL-151] - Added support for DOUBLE data type mapping in PostgreSQL
### Apache MetaModel 4.3.4
http://git-wip-us.apache.org/repos/asf/metamodel/blob/8156d76f/jdbc/src/main/java/org/apache/metamodel/jdbc/dialects/PostgresqlQueryRewriter.java
----------------------------------------------------------------------
diff --git a/jdbc/src/main/java/org/apache/metamodel/jdbc/dialects/PostgresqlQueryRewriter.java b/jdbc/src/main/java/org/apache/metamodel/jdbc/dialects/PostgresqlQueryRewriter.java
index d0b9ab5..eebb116 100644
--- a/jdbc/src/main/java/org/apache/metamodel/jdbc/dialects/PostgresqlQueryRewriter.java
+++ b/jdbc/src/main/java/org/apache/metamodel/jdbc/dialects/PostgresqlQueryRewriter.java
@@ -52,6 +52,9 @@ public class PostgresqlQueryRewriter extends LimitOffsetQueryRewriter implements
if (columnType == ColumnType.BIT) {
return "BOOLEAN";
}
+ if (columnType == ColumnType.DOUBLE) {
+ return "double precision";
+ }
return super.rewriteColumnType(columnType, columnSize);
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/8156d76f/jdbc/src/test/java/org/apache/metamodel/jdbc/integrationtests/PostgresqlTest.java
----------------------------------------------------------------------
diff --git a/jdbc/src/test/java/org/apache/metamodel/jdbc/integrationtests/PostgresqlTest.java b/jdbc/src/test/java/org/apache/metamodel/jdbc/integrationtests/PostgresqlTest.java
index f20ece4..a68cc99 100644
--- a/jdbc/src/test/java/org/apache/metamodel/jdbc/integrationtests/PostgresqlTest.java
+++ b/jdbc/src/test/java/org/apache/metamodel/jdbc/integrationtests/PostgresqlTest.java
@@ -58,6 +58,7 @@ import org.junit.Ignore;
public class PostgresqlTest extends AbstractJdbIntegrationTest {
private static final String PROPERTY_LONGRUNNINGTESTS = "jdbc.postgresql.longrunningtests";
+ private static final double DELTA = 1E-15;
@Override
protected String getPropertyPrefix() {
@@ -361,6 +362,65 @@ public class PostgresqlTest extends AbstractJdbIntegrationTest {
});
}
+ /**
+ * Tests type rewriting for double type.
+ *
+ * @see https://issues.apache.org/jira/browse/METAMODEL-151
+ */
+ public void testDouble() throws Exception {
+ if (!isConfigured()) {
+ return;
+ }
+
+ JdbcDataContext dc = new JdbcDataContext(getConnection());
+
+ final Schema schema = dc.getDefaultSchema();
+
+ dc.executeUpdate(new UpdateScript() {
+ @Override
+ public void run(UpdateCallback cb) {
+ Table table = cb.createTable(schema, "my_table").withColumn("id").ofType(ColumnType.INTEGER)
+ .ofNativeType("SERIAL").nullable(false).withColumn("some_double").ofType(ColumnType.DOUBLE)
+ .nullable(false).execute();
+ assertEquals("my_table", table.getName());
+
+ cb.insertInto(table).value("id", 1).value("some_double", Double.MIN_VALUE).execute();
+ cb.insertInto(table).value("id", 2).value("some_double", Double.MAX_VALUE).execute();
+ cb.insertInto(table).value("id", 3).value("some_double", Double.NEGATIVE_INFINITY).execute();
+ cb.insertInto(table).value("id", 4).value("some_double", Double.POSITIVE_INFINITY).execute();
+ cb.insertInto(table).value("id", 5).value("some_double", Double.NaN).execute();
+ }
+ });
+
+ try {
+ DataSet ds = dc.query().from("my_table").select("some_double").execute();
+ assertTrue(ds.next());
+ Double minVal = (Double) ds.getRow().getValue(ds.getSelectItems()[0]);
+ assertTrue(ds.next());
+ Double maxVal = (Double) ds.getRow().getValue(ds.getSelectItems()[0]);
+ assertTrue(ds.next());
+ Double negInf = (Double) ds.getRow().getValue(ds.getSelectItems()[0]);
+ assertTrue(ds.next());
+ Double posInf = (Double) ds.getRow().getValue(ds.getSelectItems()[0]);
+ assertTrue(ds.next());
+ Double nAn = (Double) ds.getRow().getValue(ds.getSelectItems()[0]);
+ assertFalse(ds.next());
+
+ assertEquals(Double.MIN_VALUE, minVal, DELTA);
+ assertEquals(Double.MAX_VALUE, maxVal, DELTA);
+ assertTrue(Double.isInfinite(negInf));
+ assertTrue(Double.isInfinite(posInf));
+ assertTrue(Double.isNaN(nAn));
+ } finally {
+ dc.executeUpdate(new UpdateScript() {
+ @Override
+ public void run(UpdateCallback cb) {
+ cb.dropTable("my_table").execute();
+ }
+ });
+ }
+ }
+
public void testBlob() throws Exception {
if (!isConfigured()) {
return;