You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2018/06/21 05:50:58 UTC
[07/31] metamodel git commit: HBase improvements: refactoring,
new unittests
HBase improvements: refactoring, new unittests
Project: http://git-wip-us.apache.org/repos/asf/metamodel/repo
Commit: http://git-wip-us.apache.org/repos/asf/metamodel/commit/3ada7eb4
Tree: http://git-wip-us.apache.org/repos/asf/metamodel/tree/3ada7eb4
Diff: http://git-wip-us.apache.org/repos/asf/metamodel/diff/3ada7eb4
Branch: refs/heads/master
Commit: 3ada7eb48e0599b507024b1b6c10be3b7cceb63f
Parents: 38d8bc3
Author: Gerard Dellemann <g....@quadient.com>
Authored: Thu May 24 16:12:20 2018 +0200
Committer: Gerard Dellemann <g....@quadient.com>
Committed: Thu May 24 16:12:20 2018 +0200
----------------------------------------------------------------------
.../org/apache/metamodel/hbase/HBaseClient.java | 164 ++++++++++++++
.../org/apache/metamodel/hbase/HBaseColumn.java | 28 ++-
.../hbase/HBaseCreateTableBuilder.java | 94 ++++++--
.../hbase/HBaseRowDeletionBuilder.java | 24 ++-
.../hbase/HBaseRowInsertionBuilder.java | 89 ++++----
.../org/apache/metamodel/hbase/HBaseTable.java | 90 ++++++--
.../metamodel/hbase/HBaseTableDropBuilder.java | 25 ++-
.../metamodel/hbase/HBaseUpdateCallback.java | 99 +++------
.../org/apache/metamodel/hbase/HBaseWriter.java | 147 -------------
.../metamodel/hbase/HBaseDataContextTest.java | 98 +++------
.../apache/metamodel/hbase/HBaseTestCase.java | 58 ++++-
.../hbase/HBaseUpdateCallbackTest.java | 213 +++++++++++++++++++
12 files changed, 742 insertions(+), 387 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java
new file mode 100644
index 0000000..1e957fc
--- /dev/null
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.hbase;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.metamodel.MetaModelException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class can perform client-operations on a HBase datastore
+ */
+public final class HBaseClient {
+
+ private static final Logger logger = LoggerFactory.getLogger(HBaseClient.class);
+
+ private final Connection _connection;
+
+ public HBaseClient(Connection connection) {
+ this._connection = connection;
+ }
+
+ /**
+ * Write a single row of values to a HBase table
+ * @param hBaseTable
+ * @param columns
+ * @param values
+ * @throws IOException
+ */
+ public void writeRow(HBaseTable hBaseTable, HBaseColumn[] columns, Object[] values) throws IOException {
+ try (final Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName()))) {
+ int indexOfIdColumn = getIndexOfIdColumn(columns);
+
+ // Create a put with the values of indexOfIdColumn as rowkey
+ final Put put = new Put(Bytes.toBytes(values[indexOfIdColumn].toString()));
+
+ // Add the other values to the put
+ for (int i = 0; i < columns.length; i++) {
+ if (i != indexOfIdColumn) {
+ put.addColumn(Bytes.toBytes(columns[i].getColumnFamily()), Bytes.toBytes(columns[i].getQualifier()),
+ Bytes.toBytes(values[i].toString()));
+ }
+ }
+ // Add the put to the table
+ table.put(put);
+ }
+ }
+
+ /**
+ * Gets the index of the ID-column
+ * Throws an {@link MetaModelException} when no ID-column is found.
+ * @param columns
+ * @return
+ */
+ private int getIndexOfIdColumn(HBaseColumn[] columns) {
+ int indexOfIdColumn = 0;
+ boolean idColumnFound = false;
+ while (!idColumnFound && indexOfIdColumn < columns.length) {
+ if (columns[indexOfIdColumn].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) {
+ idColumnFound = true;
+ } else {
+ indexOfIdColumn++;
+ }
+ }
+ if (!idColumnFound) {
+ throw new MetaModelException("The ID Column family was not found");
+ }
+ return indexOfIdColumn;
+ }
+
+ /**
+ * Delete 1 row based on the key
+ * @param hBaseTable
+ * @param key
+ * @throws IOException
+ */
+ public void deleteRow(HBaseTable hBaseTable, Object key) throws IOException {
+ try (final Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName()));) {
+ if (rowExists(table, key) == true) {
+ table.delete(new Delete(Bytes.toBytes(key.toString())));
+ } else {
+ logger.warn("Rowkey with value " + key.toString() + " doesn't exist in the table");
+ }
+ }
+ }
+
+ /**
+ * Checks in the HBase datastore if a row exists based on the key
+ * @param table
+ * @param key
+ * @return boolean
+ * @throws IOException
+ */
+ private boolean rowExists(Table table, Object key) throws IOException {
+ final Get get = new Get(Bytes.toBytes(key.toString()));
+ return !table.get(get).isEmpty();
+ }
+
+ /**
+ * Creates a HBase table based on a tableName and it's columnFamilies
+ * @param tableName
+ * @param columnFamilies
+ * @throws IOException
+ */
+ public void createTable(String tableName, Set<String> columnFamilies) throws IOException {
+ try (final Admin admin = _connection.getAdmin()) {
+ final TableName hBasetableName = TableName.valueOf(tableName);
+ final HTableDescriptor tableDescriptor = new HTableDescriptor(hBasetableName);
+ // Add all columnFamilies to the tableDescriptor.
+ for (final String columnFamilie : columnFamilies) {
+ // The ID-column isn't needed because, it will automatically be created.
+ if (!columnFamilie.equals(HBaseDataContext.FIELD_ID)) {
+ tableDescriptor.addFamily(new HColumnDescriptor(columnFamilie));
+ }
+ }
+ admin.createTable(tableDescriptor);
+ final HTableDescriptor[] tables = admin.listTables();
+ if (tables.length != 1 && Bytes.equals(hBasetableName.getName(), tables[0].getTableName().getName())) {
+ throw new IOException("Failed create of table");
+ }
+ }
+ }
+
+ /**
+ * Disable and drop a table from a HBase datastore
+ * @param tableName
+ * @throws IOException
+ */
+ public void dropTable(String tableName) throws IOException {
+ try (final Admin admin = _connection.getAdmin()) {
+ final TableName hBasetableName = TableName.valueOf(tableName);
+ admin.disableTable(hBasetableName); // A table must be disabled first, before it can be deleted
+ admin.deleteTable(hBasetableName);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java
index 067b59f..bd57c1c 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseColumn.java
@@ -18,6 +18,9 @@
*/
package org.apache.metamodel.hbase;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
import org.apache.metamodel.schema.AbstractColumn;
import org.apache.metamodel.schema.ColumnType;
import org.apache.metamodel.schema.ColumnTypeImpl;
@@ -32,32 +35,32 @@ public final class HBaseColumn extends AbstractColumn {
private final ColumnType columnType;
private final int columnNumber;
- public HBaseColumn(String columnFamily, Table table) {
+ public HBaseColumn(final String columnFamily, final Table table) {
this(columnFamily, null, table, -1);
}
- public HBaseColumn(String columnFamily, String qualifier, Table table) {
+ public HBaseColumn(final String columnFamily, final String qualifier, final Table table) {
this(columnFamily, qualifier, table, -1);
}
- public HBaseColumn(String columnFamily, Table table, int columnNumber) {
+ public HBaseColumn(final String columnFamily, final Table table, final int columnNumber) {
this(columnFamily, null, table, columnNumber);
}
- public HBaseColumn(String columnFamily, String qualifier, Table table, int columnNumber) {
+ public HBaseColumn(final String columnFamily, final String qualifier, final Table table, final int columnNumber) {
if (columnFamily == null) {
throw new IllegalArgumentException("Column family isn't allowed to be null.");
} else if (table == null) {
throw new IllegalArgumentException("Table isn't allowed to be null.");
}
-
+
this.columnFamily = columnFamily;
this.qualifier = qualifier;
this.table = table;
this.columnNumber = columnNumber;
primaryKey = HBaseDataContext.FIELD_ID.equals(columnFamily);
-
+
if (primaryKey || qualifier != null) {
columnType = new ColumnTypeImpl("BYTE[]", SuperColumnType.LITERAL_TYPE);
} else {
@@ -131,4 +134,17 @@ public final class HBaseColumn extends AbstractColumn {
public String getQuote() {
return null;
}
+
+ /**
+ * Creates a set of columnFamilies out of an array of hbaseColumns
+ * @param hbaseColumns
+ * @return {@link LinkedHashSet}
+ */
+ public static Set<String> getColumnFamilies(HBaseColumn[] hbaseColumns) {
+ final LinkedHashSet<String> columnFamilies = new LinkedHashSet<String>();
+ for (int i = 0; i < hbaseColumns.length; i++) {
+ columnFamilies.add(hbaseColumns[i].getColumnFamily());
+ }
+ return columnFamilies;
+ }
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java
index e1f1257..ffdc20a 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java
@@ -19,53 +19,103 @@
package org.apache.metamodel.hbase;
import java.io.IOException;
-import java.util.LinkedHashSet;
+import java.util.Iterator;
import java.util.Set;
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.create.AbstractTableCreationBuilder;
import org.apache.metamodel.schema.MutableSchema;
-import org.apache.metamodel.schema.MutableTable;
import org.apache.metamodel.schema.Schema;
import org.apache.metamodel.schema.Table;
import org.apache.metamodel.util.SimpleTableDef;
+/**
+ * A builder-class to create tables in a HBase datastore
+ */
public class HBaseCreateTableBuilder extends AbstractTableCreationBuilder<HBaseUpdateCallback> {
- private Set<String> columnFamilies;
+ private Set<String> _columnFamilies;
+
+ public HBaseCreateTableBuilder(HBaseUpdateCallback updateCallback, Schema schema, String name) {
+ this(updateCallback, schema, name, null);
+ }
+ /**
+ * Create a {@link HBaseCreateTableBuilder}.
+ * Throws an {@link IllegalArgumentException} if the schema isn't a {@link MutableSchema}.
+ * @param updateCallback
+ * @param schema
+ * @param name
+ * @param columnFamilies
+ */
public HBaseCreateTableBuilder(HBaseUpdateCallback updateCallback, Schema schema, String name,
- HBaseColumn[] outputColumns) {
+ Set<String> columnFamilies) {
super(updateCallback, schema, name);
if (!(schema instanceof MutableSchema)) {
- throw new IllegalArgumentException("Not a valid schema: " + schema);
- }
- columnFamilies = new LinkedHashSet<String>();
- for (int i = 0; i < outputColumns.length; i++) {
- columnFamilies.add(outputColumns[i].getColumnFamily());
+ throw new IllegalArgumentException("Not a mutable schema: " + schema);
}
+ this._columnFamilies = columnFamilies;
}
@Override
- public Table execute() throws MetaModelException {
- final MutableTable table = getTable();
- final SimpleTableDef emptyTableDef = new SimpleTableDef(table.getName(), columnFamilies.toArray(
- new String[columnFamilies.size()]));
+ public Table execute() {
+ checkColumnFamilies(_columnFamilies);
- final HBaseUpdateCallback updateCallback = (HBaseUpdateCallback) getUpdateCallback();
+ final Table table = getTable();
+ // Add the table to the datastore
try {
- final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(updateCallback
- .getConfiguration()));
- HbaseWriter.createTable(table.getName(), columnFamilies);
+ final HBaseClient hBaseClient = getUpdateCallback().getHBaseClient();
+ hBaseClient.createTable(table.getName(), _columnFamilies);
} catch (IOException e) {
- e.printStackTrace();
+ throw new MetaModelException(e);
}
- final MutableSchema schema = (MutableSchema) table.getSchema();
- schema.addTable(new HBaseTable(updateCallback.getDataContext(), emptyTableDef, schema,
- HBaseConfiguration.DEFAULT_ROW_KEY_TYPE));
- return schema.getTableByName(table.getName());
+ // Update the schema
+ addNewTableToSchema(table);
+ return getSchema().getTableByName(table.getName());
}
+ /**
+ * Check if the new table has columnFamilies and if the ID-column is included.
+ * Throws a {@link MetaModelException} if a check fails.
+ * @param columnFamilies
+ */
+ private void checkColumnFamilies(Set<String> columnFamilies) {
+ if (columnFamilies == null || columnFamilies.size() == 0) {
+ throw new MetaModelException("Creating a table without columnFamilies");
+ }
+ boolean idColumnFound = false;
+ final Iterator<String> iterator = columnFamilies.iterator();
+ while (!idColumnFound && iterator.hasNext()) {
+ if (iterator.next().equals(HBaseDataContext.FIELD_ID)) {
+ idColumnFound = true;
+ }
+ }
+ if (!idColumnFound) {
+ throw new MetaModelException("ColumnFamily: " + HBaseDataContext.FIELD_ID + " not found");
+ }
+ }
+
+ /**
+ * Set the columnFamilies
+ * @param columnFamilies
+ */
+ public void setColumnFamilies(Set<String> columnFamilies) {
+ this._columnFamilies = columnFamilies;
+ }
+
+ /**
+ * Add the new {@link Table} to the {@link MutableSchema}
+ * @param table
+ * @param updateCallback
+ * @return {@link MutableSchema}
+ */
+ private void addNewTableToSchema(final Table table) {
+ final MutableSchema schema = (MutableSchema) getSchema();
+ final SimpleTableDef emptyTableDef = new SimpleTableDef(table.getName(), _columnFamilies.toArray(
+ new String[_columnFamilies.size()]));
+ schema.addTable(new HBaseTable((HBaseDataContext) getUpdateCallback().getDataContext(), emptyTableDef, schema,
+ HBaseConfiguration.DEFAULT_ROW_KEY_TYPE));
+ }
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java
index 925537a..3ba4ef8 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowDeletionBuilder.java
@@ -18,30 +18,38 @@
*/
package org.apache.metamodel.hbase;
+import java.io.IOException;
+
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.delete.AbstractRowDeletionBuilder;
import org.apache.metamodel.schema.Table;
+/**
+ * A builder-class to delete rows in a HBase datastore
+ */
public class HBaseRowDeletionBuilder extends AbstractRowDeletionBuilder {
- private HBaseUpdateCallback _updateCallback;
+ private HBaseClient _hBaseClient;
private Object _key;
- public HBaseRowDeletionBuilder(HBaseUpdateCallback updateCallback, Table table) {
+ public HBaseRowDeletionBuilder(final HBaseClient hBaseWriter, final Table table) {
super(table);
- if (updateCallback == null) {
+ if (hBaseWriter == null) {
throw new IllegalArgumentException("UpdateCallback cannot be null");
}
- _updateCallback = updateCallback;
-
+ this._hBaseClient = hBaseWriter;
}
@Override
- public void execute() throws MetaModelException {
+ public synchronized void execute() {
if (_key == null) {
- throw new IllegalArgumentException("Key cannot be null");
+ throw new MetaModelException("Key cannot be null");
+ }
+ try {
+ _hBaseClient.deleteRow((HBaseTable) getTable(), _key);
+ } catch (IOException e) {
+ throw new MetaModelException(e);
}
- _updateCallback.deleteRow((HBaseTable) getTable(), _key);
}
public void setKey(Object _key) {
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java
index d77cd04..1c633f7 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseRowInsertionBuilder.java
@@ -18,59 +18,70 @@
*/
package org.apache.metamodel.hbase;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Set;
+
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.insert.AbstractRowInsertionBuilder;
-import org.apache.metamodel.schema.Table;
+/**
+ * A builder-class to insert rows in a HBase datastore
+ */
public class HBaseRowInsertionBuilder extends AbstractRowInsertionBuilder<HBaseUpdateCallback> {
- private final HBaseColumn[] _outputColumns;
+ private HBaseColumn[] _hbaseColumns;
- public HBaseRowInsertionBuilder(HBaseUpdateCallback updateCallback, HBaseTable table, HBaseColumn[] outputColumns) {
- super(updateCallback, table.setColumns(outputColumns));
- _outputColumns = outputColumns;
- }
-
- @Override
- public void execute() throws MetaModelException {
- checkForMatchingColumnFamilies(getTable(), _outputColumns);
- getUpdateCallback().writeRow((HBaseTable) getTable(), _outputColumns, getValues());
+ public HBaseRowInsertionBuilder(final HBaseUpdateCallback updateCallback, final HBaseTable table) {
+ super(updateCallback, table);
+ checkTable(updateCallback, table);
}
- private void checkForMatchingColumnFamilies(Table table, HBaseColumn[] outputColumns) {
- for (int i = 0; i < outputColumns.length; i++) {
- if (!outputColumns[i].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) {
- boolean matchingColumnFound = false;
- int indexOfTablesColumn = 0;
-
- while (!matchingColumnFound && indexOfTablesColumn < table.getColumnCount()) {
- if (outputColumns[i].equals(table.getColumn(indexOfTablesColumn))) {
- matchingColumnFound = true;
- } else {
- indexOfTablesColumn++;
- }
- }
-
- if (!matchingColumnFound) {
- throw new IllegalArgumentException(String.format(
- "OutputColumnFamily: %s doesn't exist in the schema of the table", outputColumns[i]
- .getColumnFamily()));
- }
- }
+ /**
+ * Check if the table exits and it's columnFamilies exist
+ * If the table doesn't exist, then a {@link MetaModelException} is thrown
+ * @param updateCallback
+ * @param tableGettingInserts
+ */
+ private void checkTable(final HBaseUpdateCallback updateCallback, final HBaseTable tableGettingInserts) {
+ final HBaseTable tableInSchema = (HBaseTable) updateCallback.getDataContext().getDefaultSchema().getTableByName(
+ tableGettingInserts.getName());
+ if (tableInSchema == null) {
+ throw new MetaModelException("Trying to insert data into table: " + tableGettingInserts.getName()
+ + ", which doesn't exist yet");
}
+ tableInSchema.checkForNotMatchingColumns(tableGettingInserts.getColumnNames());
}
- public HBaseColumn[] getOutputColumns() {
- return _outputColumns;
+ /**
+ * Set the hbaseColumns. Checks if the columnFamilies exist in the table.
+ * @param hbaseColumns a {@link IllegalArgumentException} is thrown when this parameter is null or empty
+ */
+ public void setHbaseColumns(HBaseColumn[] hbaseColumns) {
+ if (hbaseColumns == null || hbaseColumns.length == 0) {
+ throw new IllegalArgumentException("List of hbaseColumns is null or empty");
+ }
+ final Set<String> columnFamilies = HBaseColumn.getColumnFamilies(hbaseColumns);
+ final HBaseTable tableInSchema = (HBaseTable) getTable();
+ final ArrayList<String> columnFamiliesAsList = new ArrayList<String>();
+ columnFamiliesAsList.addAll(columnFamilies);
+ tableInSchema.checkForNotMatchingColumns(columnFamiliesAsList);
+ this._hbaseColumns = hbaseColumns;
}
- public void setOutputColumns(HBaseColumn[] outputColumns) {
- if (outputColumns.length != _outputColumns.length) {
- throw new IllegalArgumentException("The amount of outputColumns don't match");
+ @Override
+ public synchronized void execute() {
+ if (_hbaseColumns == null || _hbaseColumns.length == 0) {
+ throw new MetaModelException("The hbaseColumns-array is null or empty");
+ }
+ if (getValues() == null || getValues().length == 0) {
+ throw new MetaModelException("The values-array is null or empty");
}
- for (int i = 0; i < outputColumns.length; i++) {
- _outputColumns[i] = outputColumns[i];
+ try {
+ final HBaseClient hBaseClient = getUpdateCallback().getHBaseClient();
+ hBaseClient.writeRow((HBaseTable) getTable(), _hbaseColumns, getValues());
+ } catch (IOException e) {
+ throw new MetaModelException(e);
}
}
-
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java
index def7c9c..fee2f5d 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTable.java
@@ -43,49 +43,69 @@ final class HBaseTable extends MutableTable {
private final transient HBaseDataContext _dataContext;
private final transient ColumnType _defaultRowKeyColumnType;
+ /**
+ * Creates an HBaseTable. If the tableDef variable doesn't include the ID column (see {@link HBaseDataContext#FIELD_ID}).
+ * Then it's first inserted.
+ * @param dataContext
+ * @param tableDef Table definition. The tableName, columnNames and columnTypes variables are used.
+ * @param schema {@link MutableSchema} where the table belongs to.
+ * @param defaultRowKeyColumnType This variable determines the {@link ColumnType},
+ * used when the tableDef doesn't include the ID column (see {@link HBaseDataContext#FIELD_ID}).
+ */
public HBaseTable(HBaseDataContext dataContext, SimpleTableDef tableDef, MutableSchema schema,
ColumnType defaultRowKeyColumnType) {
super(tableDef.getName(), TableType.TABLE, schema);
_dataContext = dataContext;
_defaultRowKeyColumnType = defaultRowKeyColumnType;
+ // Add the columns
final String[] columnNames = tableDef.getColumnNames();
if (columnNames == null || columnNames.length == 0) {
logger.info("No user-defined columns specified for table {}. Columns will be auto-detected.");
} else {
-
- final ColumnType[] types = tableDef.getColumnTypes();
- int columnNumber = 1;
-
- for (int i = 0; i < columnNames.length; i++) {
- String columnName = columnNames[i];
- if (HBaseDataContext.FIELD_ID.equals(columnName)) {
- final ColumnType type = types[i];
- final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, type)
- .setPrimaryKey(true).setColumnNumber(columnNumber).setTable(this);
- addColumn(idColumn);
- columnNumber++;
+ final ColumnType[] columnTypes = tableDef.getColumnTypes();
+
+ // Find the ID-Column
+ boolean idColumnFound = false;
+ int indexOfIDColumn = 0;
+ while (!idColumnFound && indexOfIDColumn < columnNames.length) {
+ if (columnNames[indexOfIDColumn].equals(HBaseDataContext.FIELD_ID)) {
+ idColumnFound = true;
+ } else {
+ indexOfIDColumn++;
}
}
- if (columnNumber == 1) {
- // insert a default definition of the id column
- final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, defaultRowKeyColumnType)
- .setPrimaryKey(true).setColumnNumber(columnNumber).setTable(this);
- addColumn(idColumn);
- columnNumber++;
+ int columnNumber = indexOfIDColumn + 1; // ColumnNumbers start from 1
+
+ // Add the ID-Column, even if the column wasn't included in columnNames
+ ColumnType columnType;
+ if (idColumnFound) {
+ columnType = columnTypes[indexOfIDColumn];
+ } else {
+ columnType = defaultRowKeyColumnType;
}
+ final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, columnType)
+ .setPrimaryKey(true)
+ .setColumnNumber(columnNumber)
+ .setTable(this);
+ addColumn(idColumn);
+ // Add the other columns
for (int i = 0; i < columnNames.length; i++) {
final String columnName = columnNames[i];
-
+ if (idColumnFound) {
+ columnNumber = i + 1; // ColumnNumbers start from 1
+ } else {
+ columnNumber = i + 2; // ColumnNumbers start from 1 + the ID-column has just been created
+ }
if (!HBaseDataContext.FIELD_ID.equals(columnName)) {
- final ColumnType type = types[i];
+ final ColumnType type = columnTypes[i];
final MutableColumn column = new MutableColumn(columnName, type);
column.setTable(this);
column.setColumnNumber(columnNumber);
- columnNumber++;
addColumn(column);
+ columnNumber++;
}
}
}
@@ -100,7 +120,9 @@ final class HBaseTable extends MutableTable {
int columnNumber = 1;
final MutableColumn idColumn = new MutableColumn(HBaseDataContext.FIELD_ID, _defaultRowKeyColumnType)
- .setPrimaryKey(true).setColumnNumber(columnNumber).setTable(this);
+ .setPrimaryKey(true)
+ .setColumnNumber(columnNumber)
+ .setTable(this);
addColumn(idColumn);
columnNumber++;
@@ -124,4 +146,28 @@ final class HBaseTable extends MutableTable {
}
return columnsInternal;
}
+
+ /**
+ * Check if a list of columnNames all exist in this table
+ * If a column doesn't exist, then a {@link MetaModelException} is thrown
+ * @param columnNamesOfCheckedTable
+ */
+ public void checkForNotMatchingColumns(final List<String> columnNamesOfCheckedTable) {
+ final List<String> columnsNamesOfExistingTable = getColumnNames();
+ for (String columnNameOfCheckedTable : columnNamesOfCheckedTable) {
+ boolean matchingColumnFound = false;
+ int i = 0;
+ while (!matchingColumnFound && i < columnsNamesOfExistingTable.size()) {
+ if (columnNameOfCheckedTable.equals(columnsNamesOfExistingTable.get(i))) {
+ matchingColumnFound = true;
+ } else {
+ i++;
+ }
+ }
+ if (!matchingColumnFound) {
+ throw new MetaModelException(String.format("ColumnFamily: %s doesn't exist in the schema of the table",
+ columnNameOfCheckedTable));
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java
index 5ab38d9..374e325 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseTableDropBuilder.java
@@ -18,20 +18,39 @@
*/
package org.apache.metamodel.hbase;
+import java.io.IOException;
+
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.drop.AbstractTableDropBuilder;
+import org.apache.metamodel.schema.MutableSchema;
import org.apache.metamodel.schema.Table;
+/**
+ * A builder-class to drop tables in a HBase datastore
+ */
public class HBaseTableDropBuilder extends AbstractTableDropBuilder {
private final HBaseUpdateCallback _updateCallback;
- public HBaseTableDropBuilder(Table table, HBaseUpdateCallback updateCallback) {
+ public HBaseTableDropBuilder(final Table table, final HBaseUpdateCallback updateCallback) {
super(table);
+ if (updateCallback.getDataContext().getDefaultSchema().getTableByName(table.getName()) == null) {
+ throw new MetaModelException("Trying to delete a table that doesn't exist in the datastore.");
+ }
_updateCallback = updateCallback;
}
@Override
- public void execute() throws MetaModelException {
- _updateCallback.dropTableExecute(getTable());
+ public void execute() {
+ try {
+ // Remove from the datastore
+ final HBaseClient hBaseClient = _updateCallback.getHBaseClient();
+ final Table table = getTable();
+ hBaseClient.dropTable(table.getName());
+
+ // Remove from schema
+ ((MutableSchema) table.getSchema()).removeTable(table);
+ } catch (IOException e) {
+ throw new MetaModelException(e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java
index 39670db..e548229 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseUpdateCallback.java
@@ -18,7 +18,7 @@
*/
package org.apache.metamodel.hbase;
-import java.io.IOException;
+import java.util.Set;
import org.apache.metamodel.AbstractUpdateCallback;
import org.apache.metamodel.UpdateCallback;
@@ -26,32 +26,35 @@ import org.apache.metamodel.create.TableCreationBuilder;
import org.apache.metamodel.delete.RowDeletionBuilder;
import org.apache.metamodel.drop.TableDropBuilder;
import org.apache.metamodel.insert.RowInsertionBuilder;
-import org.apache.metamodel.schema.MutableSchema;
import org.apache.metamodel.schema.Schema;
import org.apache.metamodel.schema.Table;
+/**
+ * This class is used to build objects to do client-operations on a HBase datastore
+ */
public class HBaseUpdateCallback extends AbstractUpdateCallback implements UpdateCallback {
- private final HBaseConfiguration _configuration;
-
- private final HBaseDataContext _dataContext;
+ private final HBaseClient _hBaseClient;
public HBaseUpdateCallback(HBaseDataContext dataContext) {
super(dataContext);
- _configuration = dataContext.getConfiguration();
- _dataContext = dataContext;
+ _hBaseClient = new HBaseClient(dataContext.getConnection());
}
@Override
- public TableCreationBuilder createTable(Schema schema, String name) throws IllegalArgumentException,
- IllegalStateException {
- throw new UnsupportedOperationException(
- "Use createTable(Schema schema, String name, HBaseColumn[] outputColumns)");
+ public TableCreationBuilder createTable(Schema schema, String name) {
+ return new HBaseCreateTableBuilder(this, schema, name);
}
- public HBaseCreateTableBuilder createTable(Schema schema, String name, HBaseColumn[] outputColumns)
- throws IllegalArgumentException, IllegalStateException {
- return new HBaseCreateTableBuilder(this, schema, name, outputColumns);
+ /**
+ * Initiates the building of a table creation operation.
+ * @param schema the schema to create the table in
+ * @param name the name of the new table
+ * @param columnFamilies the columnFamilies of the new table
+ * @return {@link HBaseCreateTableBuilder}
+ */
+ public HBaseCreateTableBuilder createTable(Schema schema, String name, Set<String> columnFamilies) {
+ return new HBaseCreateTableBuilder(this, schema, name, columnFamilies);
}
@Override
@@ -60,78 +63,34 @@ public class HBaseUpdateCallback extends AbstractUpdateCallback implements Updat
}
@Override
- public TableDropBuilder dropTable(Table table) throws IllegalArgumentException, IllegalStateException,
- UnsupportedOperationException {
+ public TableDropBuilder dropTable(Table table) {
return new HBaseTableDropBuilder(table, this);
}
- public void dropTableExecute(Table table) {
- try {
- final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(_configuration));
- HbaseWriter.dropTable(table.getName());
- MutableSchema schema = (MutableSchema) table.getSchema();
- schema.removeTable(table);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
@Override
- public RowInsertionBuilder insertInto(Table table) throws IllegalArgumentException, IllegalStateException,
- UnsupportedOperationException {
- throw new UnsupportedOperationException("Use insertInto(Table table, HBaseColumn[] outputColumns)");
- }
-
- public HBaseRowInsertionBuilder insertInto(Table table, HBaseColumn[] columns) throws IllegalArgumentException {
+ public RowInsertionBuilder insertInto(Table table) {
if (table instanceof HBaseTable) {
- return new HBaseRowInsertionBuilder(this, (HBaseTable) table, columns);
- }
- throw new IllegalArgumentException("Not an HBase table: " + table);
- }
-
- protected synchronized void writeRow(HBaseTable hBaseTable, HBaseColumn[] outputColumns, Object[] values) {
- try {
- final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(_configuration));
- HbaseWriter.writeRow(hBaseTable, outputColumns, values);
- } catch (IOException e) {
- e.printStackTrace();
+ return new HBaseRowInsertionBuilder(this, (HBaseTable) table);
+ } else {
+ throw new IllegalArgumentException("Not an HBase table: " + table);
}
}
@Override
public boolean isDeleteSupported() {
- return false;
+ return true;
}
@Override
- public RowDeletionBuilder deleteFrom(Table table) throws IllegalArgumentException, IllegalStateException,
- UnsupportedOperationException {
- throw new UnsupportedOperationException(
- "Use deleteFrom(HBaseUpdateCallback updateCallback, Table table, Object key)");
- }
-
- public HBaseRowDeletionBuilder deleteFrom(HBaseUpdateCallback updateCallback, Table table)
- throws IllegalArgumentException {
+ public RowDeletionBuilder deleteFrom(Table table) {
if (table instanceof HBaseTable) {
- return new HBaseRowDeletionBuilder(this, (HBaseTable) table);
+ return new HBaseRowDeletionBuilder(_hBaseClient, (HBaseTable) table);
+ } else {
+ throw new IllegalArgumentException("Not an HBase table: " + table);
}
- throw new IllegalArgumentException("Not an HBase table: " + table);
- }
-
- protected synchronized void deleteRow(HBaseTable hBaseTable, Object key) {
- try {
- final HBaseWriter HbaseWriter = new HBaseWriter(HBaseDataContext.createConfig(_configuration));
- HbaseWriter.deleteRow(hBaseTable, key);
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
-
- public HBaseConfiguration getConfiguration() {
- return _configuration;
}
- public HBaseDataContext getDataContext() {
- return _dataContext;
+ public HBaseClient getHBaseClient() {
+ return _hBaseClient;
}
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java
----------------------------------------------------------------------
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java
deleted file mode 100644
index ad63597..0000000
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseWriter.java
+++ /dev/null
@@ -1,147 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.metamodel.hbase;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.metamodel.MetaModelException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-public final class HBaseWriter extends Configured {
-
- private static final Logger logger = LoggerFactory.getLogger(HBaseWriter.class);
-
- private final Connection _connection;
-
- public HBaseWriter(Configuration configuration) throws IOException {
- _connection = ConnectionFactory.createConnection(configuration);
- }
-
- public void writeRow(HBaseTable hBaseTable, HBaseColumn[] outputColumns, Object[] values) throws IOException {
- try {
- Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName()));
- try {
- int indexOfIdColumn = 0;
- boolean idColumnFound = false;
- while (!idColumnFound && indexOfIdColumn < outputColumns.length) {
- if (outputColumns[indexOfIdColumn].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) {
- idColumnFound = true;
- } else {
- indexOfIdColumn++;
- }
- }
- if (!idColumnFound) {
- throw new MetaModelException("The ID Column family was not found");
- }
-
- Put put = new Put(Bytes.toBytes(values[indexOfIdColumn].toString()));
-
- for (int i = 0; i < outputColumns.length; i++) {
- if (!outputColumns[i].getColumnFamily().equals(HBaseDataContext.FIELD_ID)) {
- put.addColumn(Bytes.toBytes(outputColumns[i].getColumnFamily()), Bytes.toBytes(outputColumns[i]
- .getQualifier()), Bytes.toBytes(values[i].toString()));
- }
- }
- table.put(put);
- } finally {
- table.close();
- }
- } finally {
- _connection.close();
- }
- }
-
- public void deleteRow(HBaseTable hBaseTable, Object key) throws IOException {
- try {
- Table table = _connection.getTable(TableName.valueOf(hBaseTable.getName()));
- try {
- if (rowExists(table, key) == true) {
- table.delete(new Delete(Bytes.toBytes(key.toString())));
- } else {
- logger.warn("Rowkey with value " + key.toString() + " doesn't exist in the table");
- }
- } finally {
- table.close();
- }
- } finally {
- _connection.close();
- }
- }
-
- private boolean rowExists(Table table, Object key) throws IOException {
- Get get = new Get(Bytes.toBytes(key.toString()));
- return (table.get(get).isEmpty()) == true ? false : true;
- }
-
- public void createTable(String tableName, Set<String> columnFamilies) throws IOException {
- try {
- // Create table
- Admin admin = _connection.getAdmin();
- try {
- TableName hBasetableName = TableName.valueOf(tableName);
- HTableDescriptor tableDescriptor = new HTableDescriptor(hBasetableName);
- for (String columnFamilie : columnFamilies) {
- if (!columnFamilie.equals(HBaseDataContext.FIELD_ID)) {
- tableDescriptor.addFamily(new HColumnDescriptor(columnFamilie));
- }
- }
- admin.createTable(tableDescriptor);
- HTableDescriptor[] tables = admin.listTables();
- if (tables.length != 1 && Bytes.equals(hBasetableName.getName(), tables[0].getTableName().getName())) {
- throw new IOException("Failed create of table");
- }
- } finally {
- admin.close();
- }
- } finally {
- _connection.close();
- }
-
- }
-
- public void dropTable(String tableName) throws IOException {
- try {
- Admin admin = _connection.getAdmin();
- try {
- TableName hBasetableName = TableName.valueOf(tableName);
- admin.disableTable(hBasetableName);
- admin.deleteTable(hBasetableName);
- } finally {
- admin.close();
- }
- } finally {
- _connection.close();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
----------------------------------------------------------------------
diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
index fb21b46..3872611 100644
--- a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
+++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseDataContextTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.metamodel.hbase;
+import java.io.IOException;
import java.util.Arrays;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -32,43 +33,10 @@ import org.apache.metamodel.util.SimpleTableDef;
public class HBaseDataContextTest extends HBaseTestCase {
- // Table
- private static final String EXAMPLE_TABLE_NAME = "table_for_junit";
-
- // ColumnFamilies
- private static final String CF_FOO = "foo";
- private static final String CF_BAR = "bar";
-
- // Qualifiers
- private static final String Q_HELLO = "hello";
- private static final String Q_HI = "hi";
- private static final String Q_HEY = "hey";
- private static final String Q_BAH = "bah";
-
- // RowKeys
- private static final String RK_1 = "junit1";
- private static final String RK_2 = "junit2";
-
- private static final int NUMBER_OF_ROWS = 2;
-
- // Values
- private static final String V_WORLD = "world";
- private static final String V_THERE = "there";
- private static final String V_YO = "yo";
- private static final byte[] V_123_BYTE_ARRAY = new byte[] { 1, 2, 3 };
- private static final String V_YOU = "you";
-
- private HBaseDataContext _dataContext;
-
@Override
protected void setUp() throws Exception {
super.setUp();
if (isConfigured()) {
- final String zookeeperHostname = getZookeeperHostname();
- final int zookeeperPort = getZookeeperPort();
- final HBaseConfiguration configuration = new HBaseConfiguration(zookeeperHostname, zookeeperPort,
- ColumnType.VARCHAR);
- _dataContext = new HBaseDataContext(configuration);
createTableNatively();
}
}
@@ -80,7 +48,7 @@ public class HBaseDataContextTest extends HBaseTestCase {
}
// test the schema exploration
- final Table table = _dataContext.getDefaultSchema().getTableByName(EXAMPLE_TABLE_NAME);
+ final Table table = getDataContext().getDefaultSchema().getTableByName(TABLE_NAME);
assertNotNull(table);
assertEquals("[" + HBaseDataContext.FIELD_ID + ", " + CF_BAR + ", " + CF_FOO + "]", Arrays.toString(table
@@ -92,7 +60,7 @@ public class HBaseDataContextTest extends HBaseTestCase {
insertRecordsNatively();
// query using regular configuration
- final DataSet dataSet1 = _dataContext.query().from(EXAMPLE_TABLE_NAME).selectAll().execute();
+ final DataSet dataSet1 = getDataContext().query().from(TABLE_NAME).selectAll().execute();
try {
assertTrue(dataSet1.next());
assertEquals("Row[values=[" + RK_1 + ", {" + Q_HEY + "=" + V_YO + "," + Q_HI + "=" + V_THERE + "}, {"
@@ -111,14 +79,14 @@ public class HBaseDataContextTest extends HBaseTestCase {
final String columnName3 = CF_BAR + ":" + Q_HEY;
final String[] columnNames = new String[] { columnName1, columnName2, columnName3 };
final ColumnType[] columnTypes = new ColumnType[] { ColumnType.MAP, ColumnType.VARCHAR, ColumnType.VARCHAR };
- final SimpleTableDef[] tableDefinitions = new SimpleTableDef[] { new SimpleTableDef(EXAMPLE_TABLE_NAME,
+ final SimpleTableDef[] tableDefinitions = new SimpleTableDef[] { new SimpleTableDef(TABLE_NAME,
columnNames, columnTypes) };
- _dataContext = new HBaseDataContext(new HBaseConfiguration("SCH", getZookeeperHostname(), getZookeeperPort(),
- tableDefinitions, ColumnType.VARCHAR));
+ setDataContext(new HBaseDataContext(new HBaseConfiguration("SCH", getZookeeperHostname(), getZookeeperPort(),
+ tableDefinitions, ColumnType.VARCHAR)));
- final DataSet dataSet2 = _dataContext
+ final DataSet dataSet2 = getDataContext()
.query()
- .from(EXAMPLE_TABLE_NAME)
+ .from(TABLE_NAME)
.select(columnName1, columnName2, columnName3)
.execute();
try {
@@ -134,7 +102,7 @@ public class HBaseDataContextTest extends HBaseTestCase {
}
// query count
- final DataSet dataSet3 = _dataContext.query().from(EXAMPLE_TABLE_NAME).selectCount().execute();
+ final DataSet dataSet3 = getDataContext().query().from(TABLE_NAME).selectCount().execute();
try {
assertTrue(dataSet3.next());
assertEquals("Row[values=[" + NUMBER_OF_ROWS + "]]", dataSet3.getRow().toString());
@@ -144,9 +112,9 @@ public class HBaseDataContextTest extends HBaseTestCase {
}
// query only id
- final DataSet dataSet4 = _dataContext
+ final DataSet dataSet4 = getDataContext()
.query()
- .from(EXAMPLE_TABLE_NAME)
+ .from(TABLE_NAME)
.select(HBaseDataContext.FIELD_ID)
.execute();
@@ -161,9 +129,9 @@ public class HBaseDataContextTest extends HBaseTestCase {
}
// primary key lookup query - using GET
- final DataSet dataSet5 = _dataContext
+ final DataSet dataSet5 = getDataContext()
.query()
- .from(EXAMPLE_TABLE_NAME)
+ .from(TABLE_NAME)
.select(HBaseDataContext.FIELD_ID)
.where(HBaseDataContext.FIELD_ID)
.eq(RK_1)
@@ -178,9 +146,8 @@ public class HBaseDataContextTest extends HBaseTestCase {
}
}
- private void insertRecordsNatively() throws Exception {
- final org.apache.hadoop.hbase.client.Table hTable = _dataContext.getHTable(EXAMPLE_TABLE_NAME);
- try {
+ private void insertRecordsNatively() throws IOException, InterruptedException {
+ try (final org.apache.hadoop.hbase.client.Table hTable = getDataContext().getHTable(TABLE_NAME)) {
final Put put1 = new Put(RK_1.getBytes());
put1.addColumn(CF_FOO.getBytes(), Q_HELLO.getBytes(), V_WORLD.getBytes());
put1.addColumn(CF_BAR.getBytes(), Q_HI.getBytes(), V_THERE.getBytes());
@@ -192,27 +159,26 @@ public class HBaseDataContextTest extends HBaseTestCase {
final Object[] result = new Object[NUMBER_OF_ROWS];
hTable.batch(Arrays.asList(put1, put2), result);
- } finally {
- hTable.close();
}
}
- private void createTableNatively() throws Exception {
- final TableName tableName = TableName.valueOf(EXAMPLE_TABLE_NAME);
-
- // check if the table exists
- if (_dataContext.getAdmin().isTableAvailable(tableName)) {
- System.out.println("Unittest table already exists: " + EXAMPLE_TABLE_NAME);
- // table already exists
- return;
+ private void createTableNatively() throws IOException {
+ try (Admin admin = getDataContext().getAdmin()) {
+ final TableName tableName = TableName.valueOf(TABLE_NAME);
+
+ // Check if the table exists
+ if (admin.isTableAvailable(tableName)) {
+ // table already exists
+ System.out.println("Unittest table already exists: " + TABLE_NAME);
+ } else {
+ // Create table
+ System.out.println("Creating table");
+ final HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
+ tableDescriptor.addFamily(new HColumnDescriptor(CF_FOO.getBytes()));
+ tableDescriptor.addFamily(new HColumnDescriptor(CF_BAR.getBytes()));
+ admin.createTable(tableDescriptor);
+ System.out.println("Created table");
+ }
}
-
- Admin admin = _dataContext.getAdmin();
- System.out.println("Creating table");
- final HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
- tableDescriptor.addFamily(new HColumnDescriptor(CF_FOO.getBytes()));
- tableDescriptor.addFamily(new HColumnDescriptor(CF_BAR.getBytes()));
- admin.createTable(tableDescriptor);
- System.out.println("Created table");
}
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java
----------------------------------------------------------------------
diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java
index 1f7f162..b9524a8 100644
--- a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java
+++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseTestCase.java
@@ -22,14 +22,45 @@ import java.io.File;
import java.io.FileReader;
import java.util.Properties;
+import org.apache.metamodel.schema.ColumnType;
+
import junit.framework.TestCase;
public abstract class HBaseTestCase extends TestCase {
+ // TableName
+ protected static final String TABLE_NAME = "table_for_junit";
+
+ // ColumnFamilies
+ protected static final int NUMBER_OF_CFS = 3; // foo + bar + ID
+ protected static final String CF_FOO = "foo";
+ protected static final String CF_BAR = "bar";
+
+ // Qualifiers
+ protected static final String Q_HELLO = "hello";
+ protected static final String Q_HI = "hi";
+ protected static final String Q_HEY = "hey";
+ protected static final String Q_BAH = "bah";
+
+ // Number of rows
+ protected static final int NUMBER_OF_ROWS = 2;
+
+ // RowKeys
+ protected static final String RK_1 = "junit1";
+ protected static final String RK_2 = "junit2";
+
+ // RowValues
+ protected static final String V_WORLD = "world";
+ protected static final String V_THERE = "there";
+ protected static final String V_YO = "yo";
+ protected static final byte[] V_123_BYTE_ARRAY = new byte[] { 1, 2, 3 };
+ protected static final String V_YOU = "you";
+
private String zookeeperHostname;
private int zookeeperPort;
private boolean _configured;
-
+ private HBaseDataContext _dataContext;
+
@Override
protected void setUp() throws Exception {
super.setUp();
@@ -43,13 +74,18 @@ public abstract class HBaseTestCase extends TestCase {
if (zookeeperPortPropertyValue != null && !zookeeperPortPropertyValue.isEmpty()) {
zookeeperPort = Integer.parseInt(zookeeperPortPropertyValue);
}
-
+
_configured = (zookeeperHostname != null && !zookeeperHostname.isEmpty());
} else {
_configured = false;
}
+ if (isConfigured()) {
+ final HBaseConfiguration configuration = new HBaseConfiguration(zookeeperHostname, zookeeperPort,
+ ColumnType.VARCHAR);
+ setDataContext(new HBaseDataContext(configuration));
+ }
}
-
+
private String getPropertyFilePath() {
String userHome = System.getProperty("user.home");
return userHome + "/metamodel-integrationtest-configuration.properties";
@@ -63,7 +99,7 @@ public abstract class HBaseTestCase extends TestCase {
public boolean isConfigured() {
return _configured;
}
-
+
public String getZookeeperHostname() {
return zookeeperHostname;
}
@@ -71,4 +107,18 @@ public abstract class HBaseTestCase extends TestCase {
public int getZookeeperPort() {
return zookeeperPort;
}
+
+ public HBaseDataContext getDataContext() {
+ return _dataContext;
+ }
+
+ public void setDataContext(HBaseDataContext dataContext) {
+ this._dataContext = dataContext;
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ super.tearDown();
+ _dataContext.getConnection().close();
+ }
}
http://git-wip-us.apache.org/repos/asf/metamodel/blob/3ada7eb4/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java
----------------------------------------------------------------------
diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java
new file mode 100644
index 0000000..359be01
--- /dev/null
+++ b/hbase/src/test/java/org/apache/metamodel/hbase/HBaseUpdateCallbackTest.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.metamodel.hbase;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.metamodel.MetaModelException;
+import org.apache.metamodel.schema.ColumnType;
+import org.apache.metamodel.schema.ImmutableSchema;
+import org.apache.metamodel.schema.MutableSchema;
+import org.apache.metamodel.schema.Table;
+import org.apache.metamodel.util.SimpleTableDef;
+
+public class HBaseUpdateCallbackTest extends HBaseTestCase {
+
+ private HBaseUpdateCallback updateCallback;
+ private MutableSchema schema;
+
+ @Override
+ protected void setUp() throws Exception {
+ super.setUp();
+ if (isConfigured()) {
+ updateCallback = new HBaseUpdateCallback(getDataContext());
+ schema = (MutableSchema) getDataContext().getDefaultSchema();
+
+ if (schema.getTableByName(TABLE_NAME) != null) {
+ dropTableIfItExists();
+ }
+ }
+ }
+
+ public void testDropTable() throws IOException {
+ dropTableIfItExists();
+
+ try {
+ HBaseTable table = createHBaseTable();
+ updateCallback.dropTable(table).execute();
+ fail("Should get an exception that the table doesn't exist in the datastore");
+ } catch (MetaModelException e) {
+ assertEquals("Trying to delete a table that doesn't exist in the datastore.", e.getMessage());
+ }
+ }
+
+ private void dropTableIfItExists() {
+ Table table = schema.getTableByName(TABLE_NAME);
+ if (table != null) {
+ updateCallback.dropTable(table).execute();
+ // Check schema
+ assertNull(schema.getTableByName(TABLE_NAME));
+ // Check in the datastore
+ try (Admin admin = getDataContext().getAdmin()) {
+ assertFalse(admin.tableExists(TableName.valueOf(TABLE_NAME)));
+ } catch (IOException e) {
+ fail("Should not an exception checking if the table exists");
+ }
+ }
+ }
+
+ public void testCreateTable() {
+ // Drop the table if it exists
+ dropTableIfItExists();
+
+ // Test 1: Create a table with an immutableSchema, should throw a IllegalArgumentException
+ ImmutableSchema immutableSchema = new ImmutableSchema(schema);
+ try {
+ updateCallback.createTable(immutableSchema, TABLE_NAME).execute();
+ fail("Should get an exception that the schema isn't mutable");
+ } catch (IllegalArgumentException e) {
+ assertEquals("Not a mutable schema: " + immutableSchema, e.getMessage());
+ }
+
+ // Test 2: Create a table without columnFamilies, should throw a MetaModelException
+ try {
+ updateCallback.createTable(schema, TABLE_NAME).execute();
+ fail("Should get an exception that the columnFamilies haven't been set");
+ } catch (MetaModelException e) {
+ assertEquals("Creating a table without columnFamilies", e.getMessage());
+ }
+
+ // Test 3: Create a table with columnFamilies null, should throw a MetaModelException
+ try {
+ updateCallback.createTable(schema, TABLE_NAME, null).execute();
+ fail("Should get an exception that the columnFamilies haven't been set");
+ } catch (MetaModelException e) {
+ assertEquals("Creating a table without columnFamilies", e.getMessage());
+ }
+
+ // Test 4: Create a table with columnFamilies empty, should throw a MetaModelException
+ try {
+ final LinkedHashSet<String> columnFamilies = new LinkedHashSet<String>();
+ updateCallback.createTable(schema, TABLE_NAME, columnFamilies).execute();
+ fail("Should get an exception that the columnFamilies haven't been set");
+ } catch (MetaModelException e) {
+ assertEquals("Creating a table without columnFamilies", e.getMessage());
+ }
+
+ HBaseTable table = createHBaseTable();
+
+ // Test 5: Create a table without the ID-Column, should throw a MetaModelException
+ ArrayList<HBaseColumn> hBaseColumnsAsArrayList = createListWithHBaseColumnsExcludingIDColumn(table);
+ HBaseColumn[] hBaseColumnsAsArray = convertToHBaseColumnArray(hBaseColumnsAsArrayList);
+ Set<String> columnFamilies = HBaseColumn.getColumnFamilies(hBaseColumnsAsArray);
+ try {
+ HBaseCreateTableBuilder hBaseCreateTableBuilder = (HBaseCreateTableBuilder) updateCallback.createTable(
+ schema, TABLE_NAME);
+
+ hBaseCreateTableBuilder.setColumnFamilies(columnFamilies);
+ hBaseCreateTableBuilder.execute();
+ fail("Should get an exception that the ID-colum is missing");
+ } catch (MetaModelException e) {
+ assertEquals("ColumnFamily: " + HBaseDataContext.FIELD_ID + " not found", e.getMessage());
+ }
+
+ // Test 6: Create a table including the ID-Column (columnFamilies not in constructor), should work
+ hBaseColumnsAsArrayList = createListWithHBaseColumnsIncludingIDColumn(table);
+ hBaseColumnsAsArray = convertToHBaseColumnArray(hBaseColumnsAsArrayList);
+ columnFamilies = HBaseColumn.getColumnFamilies(hBaseColumnsAsArray);
+ try {
+ HBaseCreateTableBuilder hBaseCreateTableBuilder = (HBaseCreateTableBuilder) updateCallback.createTable(
+ schema, TABLE_NAME);
+
+ hBaseCreateTableBuilder.setColumnFamilies(HBaseColumn.getColumnFamilies(hBaseColumnsAsArray));
+ hBaseCreateTableBuilder.execute();
+ checkSuccesfullyInsertedTable();
+ } catch (Exception e) {
+ fail("Should not get an exception");
+ }
+ dropTableIfItExists();
+
+ // Test 7: Create a table including the ID-Column (columnFamilies in constructor), should work
+ try {
+ updateCallback.createTable(schema, TABLE_NAME, columnFamilies).execute();
+ checkSuccesfullyInsertedTable();
+ } catch (Exception e) {
+ fail("Should not get an exception");
+ }
+ dropTableIfItExists();
+ }
+
+ private void checkSuccesfullyInsertedTable() throws IOException {
+ // Check the schema
+ assertNotNull(schema.getTableByName(TABLE_NAME));
+ // Check in the datastore
+ try (Admin admin = getDataContext().getAdmin()) {
+ assertTrue(admin.tableExists(TableName.valueOf(TABLE_NAME)));
+ } catch (IOException e) {
+ fail("Should not an exception checking if the table exists");
+ }
+ }
+
+ // public void testInsertRows() throws IOException {
+ // // Drop the table if it exists
+ // dropTableIfItExists();
+ //
+ // insertTable();
+ // }
+
+ private void insertTable() throws IOException {
+ HBaseTable table = createHBaseTable();
+ ArrayList<HBaseColumn> hBaseColumnsAsArrayList = createListWithHBaseColumnsIncludingIDColumn(table);
+ HBaseColumn[] hBaseColumnsAsArray = convertToHBaseColumnArray(hBaseColumnsAsArrayList);
+ Set<String> columnFamilies = HBaseColumn.getColumnFamilies(hBaseColumnsAsArray);
+ updateCallback.createTable(schema, TABLE_NAME, columnFamilies).execute();
+ checkSuccesfullyInsertedTable();
+ }
+
+ private HBaseTable createHBaseTable() {
+ String[] columnNames = new String[] { CF_FOO, CF_BAR };
+ ColumnType[] columnTypes = new ColumnType[] { ColumnType.STRING, ColumnType.STRING };
+ SimpleTableDef tableDef = new SimpleTableDef(TABLE_NAME, columnNames, columnTypes);
+ return new HBaseTable(getDataContext(), tableDef, schema, ColumnType.STRING);
+ }
+
+ private static ArrayList<HBaseColumn> createListWithHBaseColumnsExcludingIDColumn(final HBaseTable table) {
+ ArrayList<HBaseColumn> hbaseColumns = new ArrayList<HBaseColumn>();
+ hbaseColumns.add(new HBaseColumn(CF_FOO, Q_HELLO, table));
+ hbaseColumns.add(new HBaseColumn(CF_FOO, Q_HI, table));
+ hbaseColumns.add(new HBaseColumn(CF_BAR, Q_HEY, table));
+ hbaseColumns.add(new HBaseColumn(CF_BAR, Q_BAH, table));
+ return hbaseColumns;
+ }
+
+ private static ArrayList<HBaseColumn> createListWithHBaseColumnsIncludingIDColumn(final HBaseTable table) {
+ ArrayList<HBaseColumn> hbaseColumns = createListWithHBaseColumnsExcludingIDColumn(table);
+ hbaseColumns.add(new HBaseColumn(HBaseDataContext.FIELD_ID, table));
+ return hbaseColumns;
+ }
+
+ private static HBaseColumn[] convertToHBaseColumnArray(final ArrayList<HBaseColumn> hBaseColumnsAsArrayList) {
+ return hBaseColumnsAsArrayList.toArray(new HBaseColumn[hBaseColumnsAsArrayList.size()]);
+ }
+}