You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2014/04/04 10:50:38 UTC
git commit: METAMODEL-45: Initial changes of HBase
Repository: incubator-metamodel
Updated Branches:
refs/heads/master 828c82d5d -> f6b3373ea
METAMODEL-45: Initial changes of HBase
Project: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/commit/f6b3373e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/tree/f6b3373e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-metamodel/diff/f6b3373e
Branch: refs/heads/master
Commit: f6b3373eaa6fb444b99a727def5c80226ddacfd9
Parents: 828c82d
Author: Kasper Sørensen <i....@gmail.com>
Authored: Sun Mar 30 21:28:34 2014 +0200
Committer: Kasper Sørensen <i....@gmail.com>
Committed: Sun Mar 30 21:28:34 2014 +0200
----------------------------------------------------------------------
README | 6 +-
.../metamodel/couchdb/CouchDbDataContext.java | 34 ++++++---
.../couchdb/CouchDbDataContextTest.java | 16 ++---
...del-integrationtest-configuration.properties | 31 ++++++++
.../metamodel/hbase/HBaseDataContextTest.java | 28 ++++----
.../apache/metamodel/hbase/HBaseTestCase.java | 74 ++++++++++++++++++++
pom.xml | 1 +
.../salesforce/SalesforceTestCase.java | 17 +++--
.../metamodel/sugarcrm/SugarCrmTestCase.java | 18 +++--
9 files changed, 181 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/README
----------------------------------------------------------------------
diff --git a/README b/README
index bd62b69..1a7025a 100644
--- a/README
+++ b/README
@@ -31,4 +31,8 @@ MetaModel uses maven for build. So code can be built with:
> mvn clean install.
-
+If you also want to run integration tests:
+1) Copy the file 'example-metamodel-integrationtest-configuration.properties' to your user home.
+2) Remove the 'example-' prefix from its filename
+3) Modify the file to enable properties of the integration tests that you're interested in.
+4) Re-run "mvn clean install".
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
----------------------------------------------------------------------
diff --git a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
index 557b7e9..0400f0c 100644
--- a/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
+++ b/couchdb/src/main/java/org/apache/metamodel/couchdb/CouchDbDataContext.java
@@ -27,15 +27,6 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.codehaus.jackson.JsonNode;
-import org.ektorp.CouchDbConnector;
-import org.ektorp.CouchDbInstance;
-import org.ektorp.StreamingViewResult;
-import org.ektorp.ViewQuery;
-import org.ektorp.ViewResult.Row;
-import org.ektorp.http.HttpClient;
-import org.ektorp.http.StdHttpClient;
-import org.ektorp.impl.StdCouchDbInstance;
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.MetaModelHelper;
import org.apache.metamodel.QueryPostprocessDataContext;
@@ -51,11 +42,25 @@ import org.apache.metamodel.schema.MutableTable;
import org.apache.metamodel.schema.Schema;
import org.apache.metamodel.schema.Table;
import org.apache.metamodel.util.SimpleTableDef;
+import org.codehaus.jackson.JsonNode;
+import org.ektorp.CouchDbConnector;
+import org.ektorp.CouchDbInstance;
+import org.ektorp.DbAccessException;
+import org.ektorp.StreamingViewResult;
+import org.ektorp.ViewQuery;
+import org.ektorp.ViewResult.Row;
+import org.ektorp.http.HttpClient;
+import org.ektorp.http.StdHttpClient;
+import org.ektorp.impl.StdCouchDbInstance;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* DataContext implementation for CouchDB
*/
public class CouchDbDataContext extends QueryPostprocessDataContext implements UpdateableDataContext {
+
+ private static final Logger logger = LoggerFactory.getLogger(CouchDbDataContext.class);
public static final int DEFAULT_PORT = 5984;
@@ -118,7 +123,7 @@ public class CouchDbDataContext extends QueryPostprocessDataContext implements U
.limit(1000));
try {
final Iterator<Row> rowIterator = streamingView.iterator();
- while (rowIterator.hasNext()) {
+ while (safeHasNext(rowIterator)) {
Row row = rowIterator.next();
JsonNode doc = row.getDocAsNode();
@@ -184,6 +189,15 @@ public class CouchDbDataContext extends QueryPostprocessDataContext implements U
return tableDef;
}
+ private static boolean safeHasNext(Iterator<Row> rowIterator) {
+ try {
+ return rowIterator.hasNext();
+ } catch (DbAccessException e) {
+ logger.warn("Failed to move to next row while detecting table", e);
+ return false;
+ }
+ }
+
public CouchDbInstance getCouchDbInstance() {
return _couchDbInstance;
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
----------------------------------------------------------------------
diff --git a/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java b/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
index afb6c5b..9124717 100644
--- a/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
+++ b/couchdb/src/test/java/org/apache/metamodel/couchdb/CouchDbDataContextTest.java
@@ -71,7 +71,7 @@ public class CouchDbDataContextTest extends TestCase {
System.out.println("Running CouchDB integration tests");
serverAvailable = true;
} catch (DbAccessException e) {
- System.out.println("!!! WARNING: Skipping CouchDB tests because local server is not available");
+ System.err.println("!!! WARNING: Skipping CouchDB tests because local server is not available");
e.printStackTrace();
serverAvailable = false;
}
@@ -101,8 +101,9 @@ public class CouchDbDataContextTest extends TestCase {
connector = couchDbInstance.createConnector("test_table_map_and_list", true);
- final CouchDbDataContext dc = new CouchDbDataContext(couchDbInstance, new SimpleTableDef("test_table_map_and_list",
- new String[] { "id", "foo", "bar" }, new ColumnType[] { ColumnType.INTEGER, ColumnType.MAP, ColumnType.LIST }));
+ final CouchDbDataContext dc = new CouchDbDataContext(couchDbInstance, new SimpleTableDef(
+ "test_table_map_and_list", new String[] { "id", "foo", "bar" }, new ColumnType[] { ColumnType.INTEGER,
+ ColumnType.MAP, ColumnType.LIST }));
Table table = null;
try {
table = dc.getTableByQualifiedLabel("test_table_map_and_list");
@@ -119,14 +120,13 @@ public class CouchDbDataContextTest extends TestCase {
dc.executeUpdate(new InsertInto(table).value("id", 1).value("foo", exampleMap).value("bar", exampleList));
- DataSet ds = dc.query().from(table).select("id","foo","bar").execute();
+ DataSet ds = dc.query().from(table).select("id", "foo", "bar").execute();
assertTrue(ds.next());
Row row = ds.getRow();
assertFalse(ds.next());
ds.close();
- assertEquals(
- "Row[values=[1, {hello=[world, welt, verden], foo=bar}, [{}, {meta=model, couch=db}]]]",
+ assertEquals("Row[values=[1, {hello=[world, welt, verden], foo=bar}, [{}, {meta=model, couch=db}]]]",
row.toString());
assertTrue(row.getValue(0) instanceof Integer);
assertTrue(row.getValue(1) instanceof Map);
@@ -241,8 +241,8 @@ public class CouchDbDataContextTest extends TestCase {
idColumn.toString());
assertTrue(idColumn.isPrimaryKey());
- assertEquals("Column[name=_rev,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]", schema
- .getTableByName(TEST_DATABASE_NAME).getColumnByName("_rev").toString());
+ assertEquals("Column[name=_rev,columnNumber=1,type=VARCHAR,nullable=true,nativeType=null,columnSize=null]",
+ schema.getTableByName(TEST_DATABASE_NAME).getColumnByName("_rev").toString());
DataSet ds;
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/example-metamodel-integrationtest-configuration.properties
----------------------------------------------------------------------
diff --git a/example-metamodel-integrationtest-configuration.properties b/example-metamodel-integrationtest-configuration.properties
new file mode 100644
index 0000000..6311be9
--- /dev/null
+++ b/example-metamodel-integrationtest-configuration.properties
@@ -0,0 +1,31 @@
+#Configuration file for the integration tests of Apache MetaModel.
+#This file contains externalized properties which enable the test
+#code to connect to actual running backends for testing the real integration.
+
+# -----------
+# HOW TO USE:
+# -----------
+# To use this file, copy it into your user home directory and uncomment the lines of relevance to you
+
+# ------------------------
+# HBase module properties:
+# ------------------------
+
+#hbase.zookeeper.hostname=localhost
+#hbase.zookeeper.port=2181
+
+# -----------------------------
+# Salesforce module properties:
+# -----------------------------
+
+#salesforce.username=
+#salesforce.password=
+#salesforce.securityToken=
+
+# ---------------------------
+# SugarCRM module properties:
+# ---------------------------
+
+#sugarcrm.username=
+#sugarcrm.password=
+#sugarcrm.numberOfAccounts=
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
----------------------------------------------------------------------
diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
index e5859b4..1441b7d 100644
--- a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
+++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
@@ -20,8 +20,6 @@ package org.apache.metamodel.hbase;
import java.util.Arrays;
-import junit.framework.TestCase;
-
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -33,23 +31,31 @@ import org.apache.metamodel.schema.ColumnType;
import org.apache.metamodel.schema.Table;
import org.apache.metamodel.util.SimpleTableDef;
-public class HBaseDataContextTest extends TestCase {
+public class HBaseDataContextTest extends HBaseTestCase {
private static final String EXAMPLE_TABLE_NAME = "table_for_junit";
- private final String hostname = HBaseConfiguration.DEFAULT_ZOOKEEPER_HOSTNAME;
- private final int port = HBaseConfiguration.DEFAULT_ZOOKEEPER_PORT;
-
private HBaseDataContext _dataContext;
@Override
protected void setUp() throws Exception {
super.setUp();
- _dataContext = new HBaseDataContext(new HBaseConfiguration(hostname, port, ColumnType.VARCHAR));
- createTableNatively();
+ if (isConfigured()) {
+ final String zookeeperHostname = getZookeeperHostname();
+ final int zookeeperPort = getZookeeperPort();
+ final HBaseConfiguration configuration = new HBaseConfiguration(zookeeperHostname, zookeeperPort,
+ ColumnType.VARCHAR);
+ _dataContext = new HBaseDataContext(configuration);
+ createTableNatively();
+ }
}
public void testCreateInsertQueryAndDrop() throws Exception {
+ if (!isConfigured()) {
+ System.err.println(getInvalidConfigurationMessage());
+ return;
+ }
+
// test the schema exploration
final Table table = _dataContext.getDefaultSchema().getTableByName(EXAMPLE_TABLE_NAME);
assertNotNull(table);
@@ -80,8 +86,8 @@ public class HBaseDataContextTest extends TestCase {
final ColumnType[] columnTypes = new ColumnType[] { ColumnType.MAP, ColumnType.VARCHAR, ColumnType.VARCHAR };
final SimpleTableDef[] tableDefinitions = new SimpleTableDef[] { new SimpleTableDef(EXAMPLE_TABLE_NAME,
columnNames, columnTypes) };
- _dataContext = new HBaseDataContext(new HBaseConfiguration("SCH", hostname, port, tableDefinitions,
- ColumnType.VARCHAR));
+ _dataContext = new HBaseDataContext(new HBaseConfiguration("SCH", getZookeeperHostname(), getZookeeperPort(),
+ tableDefinitions, ColumnType.VARCHAR));
final DataSet dataSet2 = _dataContext.query().from(EXAMPLE_TABLE_NAME).select("foo", "bar:hi", "bar:hey")
.execute();
@@ -131,8 +137,6 @@ public class HBaseDataContextTest extends TestCase {
} finally {
dataSet5.close();
}
-
- // TODO: Check if really GET was used instead of SCAN
}
private void insertRecordsNatively() throws Exception {
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java
----------------------------------------------------------------------
diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java
new file mode 100644
index 0000000..1f7f162
--- /dev/null
+++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.hbase;
+
+import java.io.File;
+import java.io.FileReader;
+import java.util.Properties;
+
+import junit.framework.TestCase;
+
+public abstract class HBaseTestCase extends TestCase {
+
+ private String zookeeperHostname;
+ private int zookeeperPort;
+ private boolean _configured;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+
+ Properties properties = new Properties();
+ File file = new File(getPropertyFilePath());
+ if (file.exists()) {
+ properties.load(new FileReader(file));
+ zookeeperHostname = properties.getProperty("hbase.zookeeper.hostname");
+ String zookeeperPortPropertyValue = properties.getProperty("hbase.zookeeper.port");
+ if (zookeeperPortPropertyValue != null && !zookeeperPortPropertyValue.isEmpty()) {
+ zookeeperPort = Integer.parseInt(zookeeperPortPropertyValue);
+ }
+
+ _configured = (zookeeperHostname != null && !zookeeperHostname.isEmpty());
+ } else {
+ _configured = false;
+ }
+ }
+
+ private String getPropertyFilePath() {
+ String userHome = System.getProperty("user.home");
+ return userHome + "/metamodel-integrationtest-configuration.properties";
+ }
+
+ protected String getInvalidConfigurationMessage() {
+ return "!!! WARN !!! HBase module ignored\r\n" + "Please configure HBase properties locally ("
+ + getPropertyFilePath() + "), to run integration tests";
+ }
+
+ public boolean isConfigured() {
+ return _configured;
+ }
+
+ public String getZookeeperHostname() {
+ return zookeeperHostname;
+ }
+
+ public int getZookeeperPort() {
+ return zookeeperPort;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index e2b321d..903fb1d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -337,6 +337,7 @@
<exclude>NOTICE.txt</exclude>
<exclude>LICENSE.txt</exclude>
<exclude>KEYS</exclude>
+ <exclude>example-metamodel-integrationtest-configuration.properties</exclude>
<exclude>**/.gitignore/**</exclude>
<exclude>.git/**</exclude>
<exclude>**/src/test/resources/**</exclude>
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/salesforce/src/test/java/org/apache/metamodel/salesforce/SalesforceTestCase.java
----------------------------------------------------------------------
diff --git a/salesforce/src/test/java/org/apache/metamodel/salesforce/SalesforceTestCase.java b/salesforce/src/test/java/org/apache/metamodel/salesforce/SalesforceTestCase.java
index 5e2034f..039fd61 100644
--- a/salesforce/src/test/java/org/apache/metamodel/salesforce/SalesforceTestCase.java
+++ b/salesforce/src/test/java/org/apache/metamodel/salesforce/SalesforceTestCase.java
@@ -41,22 +41,25 @@ public abstract class SalesforceTestCase extends TestCase {
Properties properties = new Properties();
File file = new File(getPropertyFilePath());
- _configured = file.exists();
- if (_configured) {
+ if (file.exists()) {
properties.load(new FileReader(file));
- _username = properties.getProperty("username");
- _password = properties.getProperty("password");
- _securityToken = properties.getProperty("securityToken");
+ _username = properties.getProperty("salesforce.username");
+ _password = properties.getProperty("salesforce.password");
+ _securityToken = properties.getProperty("salesforce.securityToken");
+
+ _configured = (_username != null && !_username.isEmpty());
+ } else {
+ _configured = false;
}
}
private String getPropertyFilePath() {
String userHome = System.getProperty("user.home");
- return userHome + "/sfdc-credentials.properties";
+ return userHome + "/metamodel-integrationtest-configuration.properties";
}
protected String getInvalidConfigurationMessage() {
- return "!!! WARN !!! Salesforce ignored\r\n" + "Please configure salesforce credentials locally ("
+ return "!!! WARN !!! Salesforce module ignored\r\n" + "Please configure salesforce credentials locally ("
+ getPropertyFilePath() + "), to run integration tests";
}
http://git-wip-us.apache.org/repos/asf/incubator-metamodel/blob/f6b3373e/sugarcrm/src/test/java/org/apache/metamodel/sugarcrm/SugarCrmTestCase.java
----------------------------------------------------------------------
diff --git a/sugarcrm/src/test/java/org/apache/metamodel/sugarcrm/SugarCrmTestCase.java b/sugarcrm/src/test/java/org/apache/metamodel/sugarcrm/SugarCrmTestCase.java
index 7b0ee0e..51cebe9 100644
--- a/sugarcrm/src/test/java/org/apache/metamodel/sugarcrm/SugarCrmTestCase.java
+++ b/sugarcrm/src/test/java/org/apache/metamodel/sugarcrm/SugarCrmTestCase.java
@@ -41,18 +41,24 @@ public abstract class SugarCrmTestCase extends TestCase {
Properties properties = new Properties();
File file = new File(getPropertyFilePath());
- _configured = file.exists();
- if (_configured) {
+ if (file.exists()) {
properties.load(new FileReader(file));
- _username = properties.getProperty("username");
- _password = properties.getProperty("password");
- _numberOfAccounts = Integer.parseInt(properties.getProperty("number_of_accounts"));
+ _username = properties.getProperty("sugarcrm.username");
+ _password = properties.getProperty("sugarcrm.password");
+ String numberOfAccountsPropertyValue = properties.getProperty("sugarcrm.numberOfAccounts");
+ if (numberOfAccountsPropertyValue != null && !numberOfAccountsPropertyValue.isEmpty()) {
+ _numberOfAccounts = Integer.parseInt(numberOfAccountsPropertyValue);
+ }
+
+ _configured = (_username != null && !_username.isEmpty());
+ } else {
+ _configured = false;
}
}
private String getPropertyFilePath() {
String userHome = System.getProperty("user.home");
- return userHome + "/sugarcrm-credentials.properties";
+ return userHome + "/metamodel-integrationtest-configuration.properties";
}
public int getNumberOfAccounts() {