You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@metamodel.apache.org by ka...@apache.org on 2019/04/01 17:36:24 UTC
[metamodel] 01/16: Usually hbase has a lot of data to write,
in order to spread the write pressure,
the pre-partition will be created in advance,
so I add a method of creating a table with splitKey.
This is an automated email from the ASF dual-hosted git repository.
kaspersor pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/metamodel.git
commit 0c078a9d054ec8cd5a55a4a519c6446a743a8049
Author: 李小保 <li...@mininglamp.com>
AuthorDate: Fri Mar 22 11:38:45 2019 +0800
Usually hbase has a lot of data to write, in order to spread the write pressure, the pre-partition will be created in advance, so I add a method of creating a table with splitKey.
---
.../org/apache/metamodel/hbase/HBaseClient.java | 53 ++++++++++++++++------
.../metamodel/hbase/HBaseCreateTableBuilder.java | 21 +++++++--
.../apache/metamodel/hbase/CreateTableTest.java | 33 ++++++++++++++
3 files changed, 90 insertions(+), 17 deletions(-)
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java
index 2b25d84..d1cd260 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseClient.java
@@ -151,20 +151,9 @@ final class HBaseClient {
* @throws MetaModelException when a {@link IOException} is caught
*/
public void createTable(final String tableName, final Set<String> columnFamilies) {
- if (tableName == null || columnFamilies == null || columnFamilies.isEmpty()) {
- throw new IllegalArgumentException("Can't create a table without having the tableName or columnFamilies");
- }
+ checkTableAndCf(tableName, columnFamilies);
try (final Admin admin = _connection.getAdmin()) {
- final TableName hBasetableName = TableName.valueOf(tableName);
- final TableDescriptorBuilder tableBuilder = TableDescriptorBuilder.newBuilder(hBasetableName);
- // Add all columnFamilies to the tableDescriptor.
- for (final String columnFamily : columnFamilies) {
- // The ID-column isn't needed because, it will automatically be created.
- if (!columnFamily.equals(HBaseDataContext.FIELD_ID)) {
- final ColumnFamilyDescriptor columnDescriptor = ColumnFamilyDescriptorBuilder.of(columnFamily);
- tableBuilder.setColumnFamily(columnDescriptor);
- }
- }
+ final TableDescriptorBuilder tableBuilder = getTableDescriptorBuilder(tableName, columnFamilies);
admin.createTable(tableBuilder.build());
} catch (IOException e) {
throw new MetaModelException(e);
@@ -172,6 +161,44 @@ final class HBaseClient {
}
/**
+ * Creates a HBase table based on a tableName and it's columnFamilies and splitKeys
+ * @param tableName
+ * @param columnFamilies
+ * @param splitKeys
+ * @throws IllegalArgumentException when any parameter is null
+ * @throws MetaModelException when a {@link IOException} is caught
+ */
+ public void createTable(final String tableName, final Set<String> columnFamilies, byte[][] splitKeys) {
+ checkTableAndCf(tableName, columnFamilies);
+ try (final Admin admin = _connection.getAdmin()) {
+ final TableDescriptorBuilder tableBuilder = getTableDescriptorBuilder(tableName, columnFamilies);
+ admin.createTable(tableBuilder.build(),splitKeys);
+ } catch (IOException e) {
+ throw new MetaModelException(e);
+ }
+ }
+
+ private TableDescriptorBuilder getTableDescriptorBuilder(String tableName, Set<String> columnFamilies) {
+ final TableName hBasetableName = TableName.valueOf(tableName);
+ final TableDescriptorBuilder tableBuilder = TableDescriptorBuilder.newBuilder(hBasetableName);
+ // Add all columnFamilies to the tableDescriptor.
+ for (final String columnFamily : columnFamilies) {
+ // The ID-column isn't needed because, it will automatically be created.
+ if (!columnFamily.equals(HBaseDataContext.FIELD_ID)) {
+ final ColumnFamilyDescriptor columnDescriptor = ColumnFamilyDescriptorBuilder.of(columnFamily);
+ tableBuilder.setColumnFamily(columnDescriptor);
+ }
+ }
+ return tableBuilder;
+ }
+
+ private void checkTableAndCf(String tableName, Set<String> columnFamilies) {
+ if (tableName == null || columnFamilies == null || columnFamilies.isEmpty()) {
+ throw new IllegalArgumentException("Can't create a table without having the tableName or columnFamilies");
+ }
+ }
+
+ /**
* Disable and drop a table from a HBase datastore
* @param tableName
* @throws IllegalArgumentException when tableName is null
diff --git a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java
index aa722c3..37bafd4 100644
--- a/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java
+++ b/hbase/src/main/java/org/apache/metamodel/hbase/HBaseCreateTableBuilder.java
@@ -33,13 +33,14 @@ import org.apache.metamodel.util.SimpleTableDef;
*/
class HBaseCreateTableBuilder extends AbstractTableCreationBuilder<HBaseUpdateCallback> {
+ private byte[][] splitKeys;
+
/**
* Create a {@link HBaseCreateTableBuilder}.
* Throws an {@link IllegalArgumentException} if the schema isn't a {@link MutableSchema}.
* @param updateCallback
* @param schema
* @param name
- * @param columnFamilies
*/
public HBaseCreateTableBuilder(final HBaseUpdateCallback updateCallback, final Schema schema, final String name) {
super(updateCallback, schema, name);
@@ -48,6 +49,14 @@ class HBaseCreateTableBuilder extends AbstractTableCreationBuilder<HBaseUpdateCa
}
}
+ public byte[][] getSplitKeys() {
+ return splitKeys;
+ }
+
+ public void setSplitKeys(byte[][] splitKeys) {
+ this.splitKeys = splitKeys;
+ }
+
@Override
public Table execute() {
Set<String> columnFamilies = getColumnFamilies();
@@ -59,8 +68,13 @@ class HBaseCreateTableBuilder extends AbstractTableCreationBuilder<HBaseUpdateCa
final Table table = getTable();
// Add the table to the datastore
- ((HBaseDataContext) getUpdateCallback().getDataContext()).getHBaseClient().createTable(table.getName(),
- columnFamilies);
+ if (this.getSplitKeys() != null) {
+ ((HBaseDataContext) getUpdateCallback().getDataContext()).getHBaseClient().createTable(table.getName(),
+ columnFamilies,this.getSplitKeys());
+ } else {
+ ((HBaseDataContext) getUpdateCallback().getDataContext()).getHBaseClient().createTable(table.getName(),
+ columnFamilies);
+ }
// Update the schema
addNewTableToSchema(table);
@@ -87,7 +101,6 @@ class HBaseCreateTableBuilder extends AbstractTableCreationBuilder<HBaseUpdateCa
/**
* Add the new {@link Table} to the {@link MutableSchema}
* @param table
- * @param data.updateCallback
* @return {@link MutableSchema}
*/
private void addNewTableToSchema(final Table table) {
diff --git a/hbase/src/test/java/org/apache/metamodel/hbase/CreateTableTest.java b/hbase/src/test/java/org/apache/metamodel/hbase/CreateTableTest.java
index 4c71a47..c0779f9 100644
--- a/hbase/src/test/java/org/apache/metamodel/hbase/CreateTableTest.java
+++ b/hbase/src/test/java/org/apache/metamodel/hbase/CreateTableTest.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import org.apache.hadoop.hbase.TableName;
import org.apache.metamodel.MetaModelException;
import org.apache.metamodel.schema.ImmutableSchema;
import org.apache.metamodel.schema.Table;
@@ -112,6 +113,38 @@ public class CreateTableTest extends HBaseUpdateCallbackTest {
}
/**
+ * Goodflow. Create a table without the ID-Column, should work
+ *
+ * @throws IOException
+ */
+ @Test
+ public void testCreateTableWithSplitRegion() throws IOException {
+
+ int numOfRegions = 3;
+ int keyLength = 5;
+ byte[][] splitRegion = new byte[numOfRegions-1][keyLength];
+ for (int i = 0 ; i< numOfRegions-1 ; i++) {
+ splitRegion[i] = ("0000"+i).getBytes();
+ }
+
+ final HBaseCreateTableBuilder hBaseCreateTableBuilder = (HBaseCreateTableBuilder) getUpdateCallback()
+ .createTable(getSchema(), TABLE_NAME);
+
+ hBaseCreateTableBuilder.withColumn(CF_FOO);
+ hBaseCreateTableBuilder.withColumn(CF_BAR);
+ hBaseCreateTableBuilder.setSplitKeys(splitRegion);
+ hBaseCreateTableBuilder.execute();
+ checkSuccesfullyInsertedTable();
+
+ final Table table = getDataContext().getDefaultSchema().getTableByName(TABLE_NAME);
+ assertTrue(table instanceof HBaseTable);
+
+ // Assert that the Table has 3 column families, a default "_id" one, and two based on the column families for
+ // the columns.
+ assertEquals(numOfRegions,getDataContext().getAdmin().getRegions(TableName.valueOf(TABLE_NAME)).size());
+ }
+
+ /**
* Goodflow. Create a table including the ID-Column (columnFamilies not in constructor), should work
*
* @throws IOException