You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@gora.apache.org by fe...@apache.org on 2012/03/01 16:08:21 UTC
svn commit: r1295608 - in /gora/trunk: ./
gora-hbase/src/main/java/org/apache/gora/hbase/store/
Author: ferdy
Date: Thu Mar 1 15:08:20 2012
New Revision: 1295608
URL: http://svn.apache.org/viewvc?rev=1295608&view=rev
Log:
GORA-101 HBaseStore should properly support multiple tables in the mapping file.
Modified:
gora/trunk/CHANGES.txt
gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java
gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java
gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
Modified: gora/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/gora/trunk/CHANGES.txt?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/CHANGES.txt (original)
+++ gora/trunk/CHANGES.txt Thu Mar 1 15:08:20 2012
@@ -6,6 +6,8 @@ Gora Change Log
Trunk (unreleased changes):
+* GORA-101 HBaseStore should properly support multiple tables in the mapping file. (ferdy)
+
* GORA-82 Add missing license headers & RAT target to pom.xml (lewismc)
* GORA-88 HBaseByteInterface not thread safe (ferdy)
Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java (original)
+++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseColumn.java Thu Mar 1 15:08:20 2012
@@ -20,24 +20,18 @@ package org.apache.gora.hbase.store;
import java.util.Arrays;
/**
- * Stores tablename, family, qualifier tuple
+ * Store family, qualifier tuple
*/
class HBaseColumn {
- final String tableName;
final byte[] family;
final byte[] qualifier;
- public HBaseColumn(String tableName, byte[] family, byte[] qualifier) {
- this.tableName = tableName;
+ public HBaseColumn(byte[] family, byte[] qualifier) {
this.family = family==null ? null : Arrays.copyOf(family, family.length);
this.qualifier = qualifier==null ? null :
Arrays.copyOf(qualifier, qualifier.length);
}
-
- public String getTableName() {
- return tableName;
- }
/**
* @return the family (internal array returned; do not modify)
@@ -77,4 +71,12 @@ class HBaseColumn {
return false;
return true;
}
+
+ @Override
+ public String toString() {
+ return "HBaseColumn [family=" + Arrays.toString(family) + ", qualifier="
+ + Arrays.toString(qualifier) + "]";
+ }
+
+
}
Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java (original)
+++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseMapping.java Thu Mar 1 15:08:20 2012
@@ -29,35 +29,29 @@ import org.apache.hadoop.hbase.util.Byte
/**
* Mapping definitions for HBase. Thread safe.
+ * It holds a definition for a single table.
*/
public class HBaseMapping {
- private final Map<String, HTableDescriptor> tableDescriptors;
-
- //name of the primary table
- private final String tableName;
+ private final HTableDescriptor tableDescriptor;
// a map from field name to hbase column
private final Map<String, HBaseColumn> columnMap;
- public HBaseMapping(Map<String, HTableDescriptor> tableDescriptors,
- String tableName, Map<String, HBaseColumn> columnMap) {
+ public HBaseMapping(HTableDescriptor tableDescriptor,
+ Map<String, HBaseColumn> columnMap) {
super();
- this.tableDescriptors = tableDescriptors;
- this.tableName = tableName;
+ this.tableDescriptor = tableDescriptor;
this.columnMap = columnMap;
}
+
public String getTableName() {
- return tableName;
+ return tableDescriptor.getNameAsString();
}
public HTableDescriptor getTable() {
- return getTable(tableName);
- }
-
- public HTableDescriptor getTable(String tableName) {
- return tableDescriptors.get(tableName);
+ return tableDescriptor;
}
public HBaseColumn getColumn(String fieldName) {
@@ -69,18 +63,13 @@ public class HBaseMapping {
* {@link HBaseMapping} using simple immutabilty.
*
*/
- public static class HBaseMappingBuilder {
- private Map<String, HTableDescriptor> tableDescriptors
- = new HashMap<String, HTableDescriptor>();
- private String tableName;
+ public static class HBaseMappingBuilder {
+ private Map<String, Map<String, HColumnDescriptor>> tableToFamilies =
+ new HashMap<String, Map<String, HColumnDescriptor>>();
private Map<String, HBaseColumn> columnMap =
new HashMap<String, HBaseColumn>();
- public void addTable(String tableName) {
- if(!tableDescriptors.containsKey(tableName)) {
- tableDescriptors.put(tableName, new HTableDescriptor(tableName));
- }
- }
+ private String tableName;
public String getTableName() {
return tableName;
@@ -90,13 +79,28 @@ public class HBaseMapping {
this.tableName = tableName;
}
- public void addColumnFamily(String tableName, String familyName,
+ public void addFamilyProps(String tableName, String familyName,
String compression, String blockCache, String blockSize,
String bloomFilter ,String maxVersions, String timeToLive,
String inMemory) {
- HColumnDescriptor columnDescriptor = addColumnFamily(tableName,
- familyName);
+ // We keep track of all tables, because even though we
+ // only build a mapping for one table. We do this because of the way
+ // the mapping file is set up.
+ // (First family properties are defined, whereafter columns are defined).
+ //
+ // HBaseMapping in fact does not need to support multiple tables,
+ // because a Store itself only supports a single table. (Every store
+ // instance simply creates one mapping instance for itself).
+ //
+ // TODO A nice solution would be to redefine the mapping file structure.
+ // For example nest columns in families. Of course this would break compatibility.
+
+
+ Map<String, HColumnDescriptor> families = getOrCreateFamilies(tableName);;
+
+
+ HColumnDescriptor columnDescriptor = getOrCreateFamily(familyName, families);
if(compression != null)
columnDescriptor.setCompressionType(Algorithm.valueOf(compression));
@@ -112,42 +116,64 @@ public class HBaseMapping {
columnDescriptor.setTimeToLive(Integer.parseInt(timeToLive));
if(inMemory != null)
columnDescriptor.setInMemory(Boolean.parseBoolean(inMemory));
-
- getTable(tableName).addFamily(columnDescriptor);
- }
-
- public HTableDescriptor getTable(String tableName) {
- return tableDescriptors.get(tableName);
}
-
- public HColumnDescriptor addColumnFamily(String tableName,
- String familyName) {
- HTableDescriptor tableDescriptor = getTable(tableName);
- HColumnDescriptor columnDescriptor = tableDescriptor.getFamily(
- Bytes.toBytes(familyName));
- if(columnDescriptor == null) {
- columnDescriptor = new HColumnDescriptor(familyName);
- tableDescriptor.addFamily(columnDescriptor);
- }
- return columnDescriptor;
+
+ public void addColumnFamily(String tableName, String familyName) {
+ Map<String, HColumnDescriptor> families = getOrCreateFamilies(tableName);
+ getOrCreateFamily(familyName, families);
}
- public void addField(String fieldName, String tableName, String family,
- String qualifier) {
+ public void addField(String fieldName, String family, String qualifier) {
byte[] familyBytes = Bytes.toBytes(family);
byte[] qualifierBytes = qualifier == null ? null :
Bytes.toBytes(qualifier);
- HBaseColumn column = new HBaseColumn(tableName, familyBytes,
- qualifierBytes);
+ HBaseColumn column = new HBaseColumn(familyBytes, qualifierBytes);
columnMap.put(fieldName, column);
}
+
+ private HColumnDescriptor getOrCreateFamily(String familyName,
+ Map<String, HColumnDescriptor> families) {
+ HColumnDescriptor columnDescriptor = families.get(familyName);
+ if (columnDescriptor == null) {
+ columnDescriptor=new HColumnDescriptor(familyName);
+ families.put(familyName, columnDescriptor);
+ }
+ return columnDescriptor;
+ }
+
+ private Map<String, HColumnDescriptor> getOrCreateFamilies(String tableName) {
+ Map<String, HColumnDescriptor> families;
+ families = tableToFamilies.get(tableName);
+ if (families == null) {
+ families = new HashMap<String, HColumnDescriptor>();
+ tableToFamilies.put(tableName, families);
+ }
+ return families;
+ }
+
+ public void renameTable(String oldName, String newName) {
+ Map<String, HColumnDescriptor> families = tableToFamilies.remove(oldName);
+ if (families == null) throw new IllegalArgumentException(oldName + " does not exist");
+ tableToFamilies.put(newName, families);
+ }
+
/**
* @return A newly constructed mapping.
*/
public HBaseMapping build() {
- return new HBaseMapping(tableDescriptors, tableName, columnMap);
+ if (tableName == null) throw new IllegalStateException("tableName is not specified");
+
+ Map<String, HColumnDescriptor> families = tableToFamilies.get(tableName);
+ if (families == null) throw new IllegalStateException("no families for table " + tableName);
+
+ HTableDescriptor tableDescriptors = new HTableDescriptor(tableName);
+ for (HColumnDescriptor desc : families.values()) {
+ tableDescriptors.addFamily(desc);
+ }
+ return new HBaseMapping(tableDescriptors, columnMap);
}
}
+
}
\ No newline at end of file
Modified: gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java
URL: http://svn.apache.org/viewvc/gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java?rev=1295608&r1=1295607&r2=1295608&view=diff
==============================================================================
--- gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java (original)
+++ gora/trunk/gora-hbase/src/main/java/org/apache/gora/hbase/store/HBaseStore.java Thu Mar 1 15:08:20 2012
@@ -130,17 +130,18 @@ implements Configurable {
createSchema();
}
- table = new HBaseTableConnection(getConf(), mapping.getTableName(), true);
+ table = new HBaseTableConnection(getConf(), getSchemaName(), true);
}
@Override
public String getSchemaName() {
+ //return the name of this table
return mapping.getTableName();
}
@Override
public void createSchema() throws IOException {
- if(admin.tableExists(mapping.getTableName())) {
+ if(schemaExists()) {
return;
}
HTableDescriptor tableDesc = mapping.getTable();
@@ -150,11 +151,11 @@ implements Configurable {
@Override
public void deleteSchema() throws IOException {
- if(!admin.tableExists(mapping.getTableName())) {
+ if(!schemaExists()) {
return;
}
- admin.disableTable(mapping.getTableName());
- admin.deleteTable(mapping.getTableName());
+ admin.disableTable(getSchemaName());
+ admin.deleteTable(getSchemaName());
}
@Override
@@ -528,7 +529,6 @@ implements Configurable {
List<Element> tableElements = root.getChildren("table");
for(Element tableElement : tableElements) {
String tableName = tableElement.getAttributeValue("name");
- mappingBuilder.addTable(tableName);
List<Element> fieldElements = tableElement.getChildren("family");
for(Element fieldElement : fieldElements) {
@@ -540,8 +540,8 @@ implements Configurable {
String maxVersions = fieldElement.getAttributeValue("maxVersions");
String timeToLive = fieldElement.getAttributeValue("timeToLive");
String inMemory = fieldElement.getAttributeValue("inMemory");
-
- mappingBuilder.addColumnFamily(tableName, familyName, compression,
+
+ mappingBuilder.addFamilyProps(tableName, familyName, compression,
blockCache, blockSize, bloomFilter, maxVersions, timeToLive,
inMemory);
}
@@ -554,22 +554,30 @@ implements Configurable {
&& classElement.getAttributeValue("name").equals(
persistentClass.getCanonicalName())) {
- String tableName = getSchemaName(
- classElement.getAttributeValue("table"), persistentClass);
- mappingBuilder.addTable(tableName);
- mappingBuilder.setTableName(tableName);
+ String tableNameFromMapping = classElement.getAttributeValue("table");
List<Element> fields = classElement.getChildren("field");
for(Element field:fields) {
String fieldName = field.getAttributeValue("name");
String family = field.getAttributeValue("family");
String qualifier = field.getAttributeValue("qualifier");
- mappingBuilder.addField(fieldName, mappingBuilder.getTableName(),
- family, qualifier);
- mappingBuilder.addColumnFamily(mappingBuilder.getTableName(),
- family);//implicit family definition
+ mappingBuilder.addField(fieldName, family, qualifier);
+ mappingBuilder.addColumnFamily(tableNameFromMapping, family);
}
+
+ String tableName = getSchemaName(tableNameFromMapping, persistentClass);
+
+
+ if (!tableNameFromMapping.equals(tableName)) {
+ log.info("Keyclass and nameclass match but mismatching table names "
+ + " mappingfile schema is '" + tableNameFromMapping
+ + "' vs actual schema '" + tableName + "' , assuming they are the same.");
+ mappingBuilder.renameTable(tableNameFromMapping, tableName);
+ }
+ mappingBuilder.setTableName(tableName);
+ //we found a matching key and value class definition,
+ //do not continue on other class definitions
break;
}
}