You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by md...@apache.org on 2018/05/08 15:27:58 UTC
[1/3] hbase git commit: change getConf as first sequence instead of
jobContext in TableOutputFormat.checkOutputSpec, add unit tests
Repository: hbase
Updated Branches:
refs/heads/branch-2 6f2ec4639 -> 9e6871901
refs/heads/branch-2.0 9653a4d0d -> 9e3b28804
refs/heads/master 102f0bf9c -> b748ea3b0
change getConf as first sequence instead of jobContext in TableOutputFormat.checkOutputSpec, add unit tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b748ea3b
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b748ea3b
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b748ea3b
Branch: refs/heads/master
Commit: b748ea3b0d3a074b976fa0ae59fc591303e805a0
Parents: 102f0bf
Author: michael.jin <mi...@zoom.us>
Authored: Fri May 4 11:33:50 2018 +0800
Committer: Mike Drob <md...@apache.org>
Committed: Tue May 8 10:24:03 2018 -0500
----------------------------------------------------------------------
.../hbase/mapreduce/TableOutputFormat.java | 4 +-
.../hbase/spark/TableOutputFormatSuite.scala | 130 +++++++++++++++++++
2 files changed, 132 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b748ea3b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
index 4eb2654..78be5af 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -174,9 +174,9 @@ implements Configurable {
@Override
public void checkOutputSpecs(JobContext context) throws IOException,
InterruptedException {
- Configuration hConf = context.getConfiguration();
+ Configuration hConf = getConf();
if (hConf == null) {
- hConf = this.conf;
+ hConf = context.getConfiguration();
}
try (Admin admin = ConnectionFactory.createConnection(hConf).getAdmin()) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b748ea3b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/TableOutputFormatSuite.scala
----------------------------------------------------------------------
diff --git a/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/TableOutputFormatSuite.scala b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/TableOutputFormatSuite.scala
new file mode 100644
index 0000000..b306f08
--- /dev/null
+++ b/hbase-spark/src/test/scala/org/apache/hadoop/hbase/spark/TableOutputFormatSuite.scala
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.spark
+
+
+import java.text.SimpleDateFormat
+import java.util.{Date, Locale}
+
+import org.apache.hadoop.hbase.mapreduce.TableOutputFormat
+import org.apache.hadoop.hbase.{HBaseTestingUtility, TableName, TableNotFoundException}
+import org.apache.hadoop.hbase.util.Bytes
+import org.apache.hadoop.mapreduce.{Job, TaskAttemptID, TaskType}
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
+import org.apache.spark.{SparkConf, SparkContext}
+import org.scalatest.{BeforeAndAfterAll, BeforeAndAfterEach, FunSuite}
+
+import scala.util.{Failure, Success, Try}
+
+
+// Unit tests for HBASE-20521: change get configuration(TableOutputFormat.conf) object first sequence from jobContext to getConf
+// this suite contains two tests, one for normal case(getConf return null, use jobContext), create new TableOutputformat object without init TableOutputFormat.conf object,
+// configuration object inside checkOutputSpecs came from jobContext.
+// The other one(getConf return conf object) we manually call "setConf" to init TableOutputFormat.conf, for making it more straight forward, we specify a nonexistent table
+// name in conf object, checkOutputSpecs will then throw TableNotFoundException exception
+class TableOutputFormatSuite extends FunSuite with
+ BeforeAndAfterEach with BeforeAndAfterAll with Logging{
+ @transient var sc: SparkContext = null
+ var TEST_UTIL = new HBaseTestingUtility
+
+ val tableName = "TableOutputFormatTest"
+ val tableNameTest = "NonExistentTable"
+ val columnFamily = "cf"
+
+ override protected def beforeAll(): Unit = {
+ TEST_UTIL.startMiniCluster
+
+ logInfo(" - minicluster started")
+ try {
+ TEST_UTIL.deleteTable(TableName.valueOf(tableName))
+ }
+ catch {
+ case e: Exception => logInfo(" - no table " + tableName + " found")
+ }
+
+ TEST_UTIL.createTable(TableName.valueOf(tableName), Bytes.toBytes(columnFamily))
+ logInfo(" - created table")
+
+ //set "validateOutputSpecs" true anyway, force to validate output spec
+ val sparkConf = new SparkConf()
+ .setMaster("local")
+ .setAppName("test")
+
+ sc = new SparkContext(sparkConf)
+ }
+
+ override protected def afterAll(): Unit = {
+ logInfo(" - delete table: " + tableName)
+ TEST_UTIL.deleteTable(TableName.valueOf(tableName))
+ logInfo(" - shutting down minicluster")
+ TEST_UTIL.shutdownMiniCluster()
+
+ TEST_UTIL.cleanupTestDir()
+ sc.stop()
+ }
+
+ def getJobContext() = {
+ val hConf = TEST_UTIL.getConfiguration
+ hConf.set(TableOutputFormat.OUTPUT_TABLE, tableName)
+ val job = Job.getInstance(hConf)
+ job.setOutputFormatClass(classOf[TableOutputFormat[String]])
+
+ val jobTrackerId = new SimpleDateFormat("yyyyMMddHHmmss", Locale.US).format(new Date())
+ val jobAttemptId = new TaskAttemptID(jobTrackerId, 1, TaskType.MAP, 0, 0)
+ new TaskAttemptContextImpl(job.getConfiguration, jobAttemptId)
+ }
+
+ // Mock up jobContext object and execute actions in "write" function
+ // from "org.apache.spark.internal.io.SparkHadoopMapReduceWriter"
+ // this case should run normally without any exceptions
+ test("TableOutputFormat.checkOutputSpecs test without setConf called, should return true and without exceptions") {
+ val jobContext = getJobContext()
+ val format = jobContext.getOutputFormatClass
+ val jobFormat = format.newInstance
+ Try {
+ jobFormat.checkOutputSpecs(jobContext)
+ } match {
+ case Success(_) => assert(true)
+ case Failure(_) => assert(false)
+ }
+ }
+
+ // Set configuration externally, checkOutputSpec should use configuration object set by "SetConf" method
+ // rather than jobContext, this case should throw "TableNotFoundException" exception
+ test("TableOutputFormat.checkOutputSpecs test without setConf called, should throw TableNotFoundException") {
+ val jobContext = getJobContext()
+ val format = jobContext.getOutputFormatClass
+ val jobFormat = format.newInstance
+
+ val hConf = TEST_UTIL.getConfiguration
+ hConf.set(TableOutputFormat.OUTPUT_TABLE, tableNameTest)
+ jobFormat.asInstanceOf[TableOutputFormat[String]].setConf(hConf)
+ Try {
+ jobFormat.checkOutputSpecs(jobContext)
+ } match {
+ case Success(_) => assert(false)
+ case Failure(e: Exception) => {
+ if(e.isInstanceOf[TableNotFoundException])
+ assert(true)
+ else
+ assert(false)
+ }
+ }
+ }
+
+}
[3/3] hbase git commit: change getConf as first sequence instead of
jobContext in TableOutputFormat.checkOutputSpec, add unit tests
Posted by md...@apache.org.
change getConf as first sequence instead of jobContext in TableOutputFormat.checkOutputSpec, add unit tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9e3b2880
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9e3b2880
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9e3b2880
Branch: refs/heads/branch-2.0
Commit: 9e3b28804d28ad586d12df24a2e5417c25cff857
Parents: 9653a4d
Author: michael.jin <mi...@zoom.us>
Authored: Fri May 4 11:33:50 2018 +0800
Committer: Mike Drob <md...@apache.org>
Committed: Tue May 8 10:27:26 2018 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9e3b2880/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
index 0a1928b..78be5af 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -174,9 +174,9 @@ implements Configurable {
@Override
public void checkOutputSpecs(JobContext context) throws IOException,
InterruptedException {
- Configuration hConf = context.getConfiguration();
- if(hConf == null) {
- hConf = this.conf;
+ Configuration hConf = getConf();
+ if (hConf == null) {
+ hConf = context.getConfiguration();
}
try (Admin admin = ConnectionFactory.createConnection(hConf).getAdmin()) {
[2/3] hbase git commit: change getConf as first sequence instead of
jobContext in TableOutputFormat.checkOutputSpec, add unit tests
Posted by md...@apache.org.
change getConf as first sequence instead of jobContext in TableOutputFormat.checkOutputSpec, add unit tests
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9e687190
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9e687190
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9e687190
Branch: refs/heads/branch-2
Commit: 9e68719014a62f37e7559329e44e2df49738ef6c
Parents: 6f2ec46
Author: michael.jin <mi...@zoom.us>
Authored: Fri May 4 11:33:50 2018 +0800
Committer: Mike Drob <md...@apache.org>
Committed: Tue May 8 10:25:56 2018 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9e687190/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
----------------------------------------------------------------------
diff --git a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
index 0a1928b..78be5af 100644
--- a/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
+++ b/hbase-mapreduce/src/main/java/org/apache/hadoop/hbase/mapreduce/TableOutputFormat.java
@@ -174,9 +174,9 @@ implements Configurable {
@Override
public void checkOutputSpecs(JobContext context) throws IOException,
InterruptedException {
- Configuration hConf = context.getConfiguration();
- if(hConf == null) {
- hConf = this.conf;
+ Configuration hConf = getConf();
+ if (hConf == null) {
+ hConf = context.getConfiguration();
}
try (Admin admin = ConnectionFactory.createConnection(hConf).getAdmin()) {