You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sqoop.apache.org by ja...@apache.org on 2013/02/08 03:49:21 UTC
git commit: SQOOP-870: Allow Sqoop to import row key column into HBase
Updated Branches:
refs/heads/trunk 20b16fd4b -> d4d262591
SQOOP-870: Allow Sqoop to import row key column into HBase
(David Robson via Jarek Jarcec Cecho)
Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/d4d26259
Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/d4d26259
Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/d4d26259
Branch: refs/heads/trunk
Commit: d4d262591c342c863c02a9330e8e48b4be8cdf4b
Parents: 20b16fd
Author: Jarek Jarcec Cecho <ja...@apache.org>
Authored: Thu Feb 7 18:48:34 2013 -0800
Committer: Jarek Jarcec Cecho <ja...@apache.org>
Committed: Thu Feb 7 18:48:34 2013 -0800
----------------------------------------------------------------------
src/docs/user/import.txt | 6 +++
.../org/apache/sqoop/hbase/HBasePutProcessor.java | 13 +++++-
.../apache/sqoop/hbase/ToStringPutTransformer.java | 3 +-
.../com/cloudera/sqoop/hbase/HBaseImportTest.java | 34 +++++++++++++++
4 files changed, 54 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d4d26259/src/docs/user/import.txt
----------------------------------------------------------------------
diff --git a/src/docs/user/import.txt b/src/docs/user/import.txt
index 9bc4fc9..ee10b1c 100644
--- a/src/docs/user/import.txt
+++ b/src/docs/user/import.txt
@@ -607,6 +607,12 @@ Argument Description
exponent component (0.0000001); while \
a value of +false+ will use toString \
which may include an exponent (1E-7)
++sqoop.hbase.add.row.key+ When set to +false+ (default), Sqoop \
+ will not add the column used as a row \
+ key into the row data in HBase. When \
+ set to +true+, the column used as a \
+ row key will be added to the row data \
+ in HBase.
-------------------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d4d26259/src/java/org/apache/sqoop/hbase/HBasePutProcessor.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/hbase/HBasePutProcessor.java b/src/java/org/apache/sqoop/hbase/HBasePutProcessor.java
index ffa5f63..6aca97f 100644
--- a/src/java/org/apache/sqoop/hbase/HBasePutProcessor.java
+++ b/src/java/org/apache/sqoop/hbase/HBasePutProcessor.java
@@ -66,6 +66,12 @@ public class HBasePutProcessor implements Closeable, Configurable,
public static final String TRANSFORMER_CLASS_KEY =
"sqoop.hbase.insert.put.transformer.class";
+ /** Configuration key to specify whether to add the row key column into
+ * HBase. Set to false by default.
+ */
+ public static final String ADD_ROW_KEY = "sqoop.hbase.add.row.key";
+ public static final boolean ADD_ROW_KEY_DEFAULT = false;
+
private Configuration conf;
// An object that can transform a map of fieldName->object
@@ -98,9 +104,14 @@ public class HBasePutProcessor implements Closeable, Configurable,
this.putTransformer.setRowKeyColumn(conf.get(ROW_KEY_COLUMN_KEY, null));
if (this.putTransformer instanceof ToStringPutTransformer) {
- ((ToStringPutTransformer) this.putTransformer).bigDecimalFormatString =
+ ToStringPutTransformer stringPutTransformer =
+ (ToStringPutTransformer) this.putTransformer;
+ stringPutTransformer.bigDecimalFormatString =
conf.getBoolean(ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT,
ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT);
+ stringPutTransformer.addRowKey =
+ conf.getBoolean(HBasePutProcessor.ADD_ROW_KEY,
+ HBasePutProcessor.ADD_ROW_KEY_DEFAULT);
}
this.tableName = conf.get(TABLE_NAME_KEY, null);
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d4d26259/src/java/org/apache/sqoop/hbase/ToStringPutTransformer.java
----------------------------------------------------------------------
diff --git a/src/java/org/apache/sqoop/hbase/ToStringPutTransformer.java b/src/java/org/apache/sqoop/hbase/ToStringPutTransformer.java
index 131fd43..13c765c 100644
--- a/src/java/org/apache/sqoop/hbase/ToStringPutTransformer.java
+++ b/src/java/org/apache/sqoop/hbase/ToStringPutTransformer.java
@@ -44,6 +44,7 @@ public class ToStringPutTransformer extends PutTransformer {
// Used to cache serialization work done for fields names.
private Map<String, byte[]> serializedFieldNames;
protected boolean bigDecimalFormatString;
+ protected boolean addRowKey;
public ToStringPutTransformer() {
serializedFieldNames = new TreeMap<String, byte[]>();
@@ -87,7 +88,7 @@ public class ToStringPutTransformer extends PutTransformer {
for (Map.Entry<String, Object> fieldEntry : fields.entrySet()) {
String colName = fieldEntry.getKey();
- if (!colName.equals(rowKeyCol)) {
+ if (!colName.equals(rowKeyCol) || addRowKey) {
// This is a regular field, not the row key.
// Add it if it's not null.
Object val = fieldEntry.getValue();
http://git-wip-us.apache.org/repos/asf/sqoop/blob/d4d26259/src/test/com/cloudera/sqoop/hbase/HBaseImportTest.java
----------------------------------------------------------------------
diff --git a/src/test/com/cloudera/sqoop/hbase/HBaseImportTest.java b/src/test/com/cloudera/sqoop/hbase/HBaseImportTest.java
index e1f9696..d411f3d 100644
--- a/src/test/com/cloudera/sqoop/hbase/HBaseImportTest.java
+++ b/src/test/com/cloudera/sqoop/hbase/HBaseImportTest.java
@@ -127,4 +127,38 @@ public class HBaseImportTest extends HBaseTestCase {
int rowCount = countHBaseTable("nullRowT", "nullRowF");
assertEquals(0, rowCount);
}
+
+ @Test
+ public void testAddRowKey() throws IOException {
+ String[] types = { "INT", "INT" };
+ String[] vals = { "0", "1" };
+ createTableWithColTypes(types, vals);
+
+ String[] otherArg = getArgv(true, "addRowKeyT", "addRowKeyF", true, null);
+ String[] argv = new String[otherArg.length + 2];
+ argv[0] = "-D";
+ argv[1] = "sqoop.hbase.add.row.key=true";
+ System.arraycopy(otherArg, 0, argv, 2, otherArg.length);
+
+ runImport(argv);
+
+ // Row key should have been added
+ verifyHBaseCell("addRowKeyT", "0", "addRowKeyF", getColName(0), "0");
+ verifyHBaseCell("addRowKeyT", "0", "addRowKeyF", getColName(1), "1");
+ }
+
+ @Test
+ public void testAddRowKeyDefault() throws IOException {
+ String[] types = { "INT", "INT" };
+ String[] vals = { "0", "1" };
+ createTableWithColTypes(types, vals);
+
+ String[] argv = getArgv(true, "addRowKeyDfT", "addRowKeyDfF", true, null);
+
+ runImport(argv);
+
+ // Row key should not be added by default
+ verifyHBaseCell("addRowKeyDfT", "0", "addRowKeyDfF", getColName(0), null);
+ verifyHBaseCell("addRowKeyDfT", "0", "addRowKeyDfF", getColName(1), "1");
+ }
}