You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mi...@apache.org on 2016/02/06 02:34:16 UTC
[05/17] hbase-site git commit: Published site at
9c832109458ced70d0db701e7f58ef41b1df0fc7.
http://git-wip-us.apache.org/repos/asf/hbase-site/blob/9bc06f81/testdevapidocs/src-html/org/apache/hadoop/hbase/mapreduce/TestImportTsv.html
----------------------------------------------------------------------
diff --git a/testdevapidocs/src-html/org/apache/hadoop/hbase/mapreduce/TestImportTsv.html b/testdevapidocs/src-html/org/apache/hadoop/hbase/mapreduce/TestImportTsv.html
index 4d1025b..ead7bbb 100644
--- a/testdevapidocs/src-html/org/apache/hadoop/hbase/mapreduce/TestImportTsv.html
+++ b/testdevapidocs/src-html/org/apache/hadoop/hbase/mapreduce/TestImportTsv.html
@@ -371,199 +371,212 @@
<span class="sourceLineNo">363</span> doMROnTableTest(util, tn, FAMILY, data, args, 1, 4);<a name="line.363"></a>
<span class="sourceLineNo">364</span> util.deleteTable(tn);<a name="line.364"></a>
<span class="sourceLineNo">365</span> }<a name="line.365"></a>
-<span class="sourceLineNo">366</span><a name="line.366"></a>
-<span class="sourceLineNo">367</span> private Tool doMROnTableTest(String data, int valueMultiplier) throws Exception {<a name="line.367"></a>
-<span class="sourceLineNo">368</span> return doMROnTableTest(util, tn, FAMILY, data, args, valueMultiplier,-1);<a name="line.368"></a>
-<span class="sourceLineNo">369</span> }<a name="line.369"></a>
-<span class="sourceLineNo">370</span><a name="line.370"></a>
-<span class="sourceLineNo">371</span> protected static Tool doMROnTableTest(HBaseTestingUtility util, TableName table,<a name="line.371"></a>
-<span class="sourceLineNo">372</span> String family, String data, Map<String, String> args) throws Exception {<a name="line.372"></a>
-<span class="sourceLineNo">373</span> return doMROnTableTest(util, table, family, data, args, 1,-1);<a name="line.373"></a>
-<span class="sourceLineNo">374</span> }<a name="line.374"></a>
-<span class="sourceLineNo">375</span><a name="line.375"></a>
-<span class="sourceLineNo">376</span> /**<a name="line.376"></a>
-<span class="sourceLineNo">377</span> * Run an ImportTsv job and perform basic validation on the results.<a name="line.377"></a>
-<span class="sourceLineNo">378</span> * Returns the ImportTsv <code>Tool</code> instance so that other tests can<a name="line.378"></a>
-<span class="sourceLineNo">379</span> * inspect it for further validation as necessary. This method is static to<a name="line.379"></a>
-<span class="sourceLineNo">380</span> * insure non-reliance on instance's util/conf facilities.<a name="line.380"></a>
-<span class="sourceLineNo">381</span> * @param args Any arguments to pass BEFORE inputFile path is appended.<a name="line.381"></a>
-<span class="sourceLineNo">382</span> * @return The Tool instance used to run the test.<a name="line.382"></a>
-<span class="sourceLineNo">383</span> */<a name="line.383"></a>
+<span class="sourceLineNo">366</span> <a name="line.366"></a>
+<span class="sourceLineNo">367</span> @Test<a name="line.367"></a>
+<span class="sourceLineNo">368</span> public void testSkipEmptyColumns() throws Exception {<a name="line.368"></a>
+<span class="sourceLineNo">369</span> Path bulkOutputPath = new Path(util.getDataTestDirOnTestFS(tn.getNameAsString()), "hfiles");<a name="line.369"></a>
+<span class="sourceLineNo">370</span> args.put(ImportTsv.BULK_OUTPUT_CONF_KEY, bulkOutputPath.toString());<a name="line.370"></a>
+<span class="sourceLineNo">371</span> args.put(ImportTsv.COLUMNS_CONF_KEY, "HBASE_ROW_KEY,HBASE_TS_KEY,FAM:A,FAM:B");<a name="line.371"></a>
+<span class="sourceLineNo">372</span> args.put(ImportTsv.SEPARATOR_CONF_KEY, ",");<a name="line.372"></a>
+<span class="sourceLineNo">373</span> args.put(ImportTsv.SKIP_EMPTY_COLUMNS, "true");<a name="line.373"></a>
+<span class="sourceLineNo">374</span> // 2 Rows of data as input. Both rows are valid and only 3 columns are no-empty among 4<a name="line.374"></a>
+<span class="sourceLineNo">375</span> String data = "KEY,1234,VALUE1,VALUE2\nKEY,1235,,VALUE2\n";<a name="line.375"></a>
+<span class="sourceLineNo">376</span> doMROnTableTest(util, tn, FAMILY, data, args, 1, 3);<a name="line.376"></a>
+<span class="sourceLineNo">377</span> util.deleteTable(tn);<a name="line.377"></a>
+<span class="sourceLineNo">378</span> }<a name="line.378"></a>
+<span class="sourceLineNo">379</span><a name="line.379"></a>
+<span class="sourceLineNo">380</span> private Tool doMROnTableTest(String data, int valueMultiplier) throws Exception {<a name="line.380"></a>
+<span class="sourceLineNo">381</span> return doMROnTableTest(util, tn, FAMILY, data, args, valueMultiplier,-1);<a name="line.381"></a>
+<span class="sourceLineNo">382</span> }<a name="line.382"></a>
+<span class="sourceLineNo">383</span><a name="line.383"></a>
<span class="sourceLineNo">384</span> protected static Tool doMROnTableTest(HBaseTestingUtility util, TableName table,<a name="line.384"></a>
-<span class="sourceLineNo">385</span> String family, String data, Map<String, String> args, int valueMultiplier,int expectedKVCount)<a name="line.385"></a>
-<span class="sourceLineNo">386</span> throws Exception {<a name="line.386"></a>
-<span class="sourceLineNo">387</span> Configuration conf = new Configuration(util.getConfiguration());<a name="line.387"></a>
+<span class="sourceLineNo">385</span> String family, String data, Map<String, String> args) throws Exception {<a name="line.385"></a>
+<span class="sourceLineNo">386</span> return doMROnTableTest(util, table, family, data, args, 1,-1);<a name="line.386"></a>
+<span class="sourceLineNo">387</span> }<a name="line.387"></a>
<span class="sourceLineNo">388</span><a name="line.388"></a>
-<span class="sourceLineNo">389</span> // populate input file<a name="line.389"></a>
-<span class="sourceLineNo">390</span> FileSystem fs = FileSystem.get(conf);<a name="line.390"></a>
-<span class="sourceLineNo">391</span> Path inputPath = fs.makeQualified(<a name="line.391"></a>
-<span class="sourceLineNo">392</span> new Path(util.getDataTestDirOnTestFS(table.getNameAsString()), "input.dat"));<a name="line.392"></a>
-<span class="sourceLineNo">393</span> FSDataOutputStream op = fs.create(inputPath, true);<a name="line.393"></a>
-<span class="sourceLineNo">394</span> if (data == null) {<a name="line.394"></a>
-<span class="sourceLineNo">395</span> data = "KEY\u001bVALUE1\u001bVALUE2\n";<a name="line.395"></a>
-<span class="sourceLineNo">396</span> }<a name="line.396"></a>
-<span class="sourceLineNo">397</span> op.write(Bytes.toBytes(data));<a name="line.397"></a>
-<span class="sourceLineNo">398</span> op.close();<a name="line.398"></a>
-<span class="sourceLineNo">399</span> LOG.debug(String.format("Wrote test data to file: %s", inputPath));<a name="line.399"></a>
-<span class="sourceLineNo">400</span><a name="line.400"></a>
-<span class="sourceLineNo">401</span> if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {<a name="line.401"></a>
-<span class="sourceLineNo">402</span> LOG.debug("Forcing combiner.");<a name="line.402"></a>
-<span class="sourceLineNo">403</span> conf.setInt("mapreduce.map.combine.minspills", 1);<a name="line.403"></a>
-<span class="sourceLineNo">404</span> }<a name="line.404"></a>
-<span class="sourceLineNo">405</span><a name="line.405"></a>
-<span class="sourceLineNo">406</span> // Build args array.<a name="line.406"></a>
-<span class="sourceLineNo">407</span> String[] argsArray = new String[args.size() + 2];<a name="line.407"></a>
-<span class="sourceLineNo">408</span> Iterator it = args.entrySet().iterator();<a name="line.408"></a>
-<span class="sourceLineNo">409</span> int i = 0;<a name="line.409"></a>
-<span class="sourceLineNo">410</span> while (it.hasNext()) {<a name="line.410"></a>
-<span class="sourceLineNo">411</span> Map.Entry pair = (Map.Entry) it.next();<a name="line.411"></a>
-<span class="sourceLineNo">412</span> argsArray[i] = "-D" + pair.getKey() + "=" + pair.getValue();<a name="line.412"></a>
-<span class="sourceLineNo">413</span> i++;<a name="line.413"></a>
-<span class="sourceLineNo">414</span> }<a name="line.414"></a>
-<span class="sourceLineNo">415</span> argsArray[i] = table.getNameAsString();<a name="line.415"></a>
-<span class="sourceLineNo">416</span> argsArray[i + 1] = inputPath.toString();<a name="line.416"></a>
-<span class="sourceLineNo">417</span><a name="line.417"></a>
-<span class="sourceLineNo">418</span> // run the import<a name="line.418"></a>
-<span class="sourceLineNo">419</span> Tool tool = new ImportTsv();<a name="line.419"></a>
-<span class="sourceLineNo">420</span> LOG.debug("Running ImportTsv with arguments: " + argsArray);<a name="line.420"></a>
-<span class="sourceLineNo">421</span> assertEquals(0, ToolRunner.run(conf, tool, argsArray));<a name="line.421"></a>
-<span class="sourceLineNo">422</span><a name="line.422"></a>
-<span class="sourceLineNo">423</span> // Perform basic validation. If the input args did not include<a name="line.423"></a>
-<span class="sourceLineNo">424</span> // ImportTsv.BULK_OUTPUT_CONF_KEY then validate data in the table.<a name="line.424"></a>
-<span class="sourceLineNo">425</span> // Otherwise, validate presence of hfiles.<a name="line.425"></a>
-<span class="sourceLineNo">426</span> boolean isDryRun = args.containsKey(ImportTsv.DRY_RUN_CONF_KEY) &&<a name="line.426"></a>
-<span class="sourceLineNo">427</span> "true".equalsIgnoreCase(args.get(ImportTsv.DRY_RUN_CONF_KEY));<a name="line.427"></a>
-<span class="sourceLineNo">428</span> if (args.containsKey(ImportTsv.BULK_OUTPUT_CONF_KEY)) {<a name="line.428"></a>
-<span class="sourceLineNo">429</span> if (isDryRun) {<a name="line.429"></a>
-<span class="sourceLineNo">430</span> assertFalse(String.format("Dry run mode, %s should not have been created.",<a name="line.430"></a>
-<span class="sourceLineNo">431</span> ImportTsv.BULK_OUTPUT_CONF_KEY),<a name="line.431"></a>
-<span class="sourceLineNo">432</span> fs.exists(new Path(ImportTsv.BULK_OUTPUT_CONF_KEY)));<a name="line.432"></a>
-<span class="sourceLineNo">433</span> } else {<a name="line.433"></a>
-<span class="sourceLineNo">434</span> validateHFiles(fs, args.get(ImportTsv.BULK_OUTPUT_CONF_KEY), family,expectedKVCount);<a name="line.434"></a>
-<span class="sourceLineNo">435</span> }<a name="line.435"></a>
-<span class="sourceLineNo">436</span> } else {<a name="line.436"></a>
-<span class="sourceLineNo">437</span> validateTable(conf, table, family, valueMultiplier, isDryRun);<a name="line.437"></a>
-<span class="sourceLineNo">438</span> }<a name="line.438"></a>
-<span class="sourceLineNo">439</span><a name="line.439"></a>
-<span class="sourceLineNo">440</span> if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {<a name="line.440"></a>
-<span class="sourceLineNo">441</span> LOG.debug("Deleting test subdirectory");<a name="line.441"></a>
-<span class="sourceLineNo">442</span> util.cleanupDataTestDirOnTestFS(table.getNameAsString());<a name="line.442"></a>
-<span class="sourceLineNo">443</span> }<a name="line.443"></a>
-<span class="sourceLineNo">444</span> return tool;<a name="line.444"></a>
-<span class="sourceLineNo">445</span> }<a name="line.445"></a>
-<span class="sourceLineNo">446</span><a name="line.446"></a>
-<span class="sourceLineNo">447</span> /**<a name="line.447"></a>
-<span class="sourceLineNo">448</span> * Confirm ImportTsv via data in online table.<a name="line.448"></a>
-<span class="sourceLineNo">449</span> */<a name="line.449"></a>
-<span class="sourceLineNo">450</span> private static void validateTable(Configuration conf, TableName tableName,<a name="line.450"></a>
-<span class="sourceLineNo">451</span> String family, int valueMultiplier, boolean isDryRun) throws IOException {<a name="line.451"></a>
+<span class="sourceLineNo">389</span> /**<a name="line.389"></a>
+<span class="sourceLineNo">390</span> * Run an ImportTsv job and perform basic validation on the results.<a name="line.390"></a>
+<span class="sourceLineNo">391</span> * Returns the ImportTsv <code>Tool</code> instance so that other tests can<a name="line.391"></a>
+<span class="sourceLineNo">392</span> * inspect it for further validation as necessary. This method is static to<a name="line.392"></a>
+<span class="sourceLineNo">393</span> * insure non-reliance on instance's util/conf facilities.<a name="line.393"></a>
+<span class="sourceLineNo">394</span> * @param args Any arguments to pass BEFORE inputFile path is appended.<a name="line.394"></a>
+<span class="sourceLineNo">395</span> * @return The Tool instance used to run the test.<a name="line.395"></a>
+<span class="sourceLineNo">396</span> */<a name="line.396"></a>
+<span class="sourceLineNo">397</span> protected static Tool doMROnTableTest(HBaseTestingUtility util, TableName table,<a name="line.397"></a>
+<span class="sourceLineNo">398</span> String family, String data, Map<String, String> args, int valueMultiplier,int expectedKVCount)<a name="line.398"></a>
+<span class="sourceLineNo">399</span> throws Exception {<a name="line.399"></a>
+<span class="sourceLineNo">400</span> Configuration conf = new Configuration(util.getConfiguration());<a name="line.400"></a>
+<span class="sourceLineNo">401</span><a name="line.401"></a>
+<span class="sourceLineNo">402</span> // populate input file<a name="line.402"></a>
+<span class="sourceLineNo">403</span> FileSystem fs = FileSystem.get(conf);<a name="line.403"></a>
+<span class="sourceLineNo">404</span> Path inputPath = fs.makeQualified(<a name="line.404"></a>
+<span class="sourceLineNo">405</span> new Path(util.getDataTestDirOnTestFS(table.getNameAsString()), "input.dat"));<a name="line.405"></a>
+<span class="sourceLineNo">406</span> FSDataOutputStream op = fs.create(inputPath, true);<a name="line.406"></a>
+<span class="sourceLineNo">407</span> if (data == null) {<a name="line.407"></a>
+<span class="sourceLineNo">408</span> data = "KEY\u001bVALUE1\u001bVALUE2\n";<a name="line.408"></a>
+<span class="sourceLineNo">409</span> }<a name="line.409"></a>
+<span class="sourceLineNo">410</span> op.write(Bytes.toBytes(data));<a name="line.410"></a>
+<span class="sourceLineNo">411</span> op.close();<a name="line.411"></a>
+<span class="sourceLineNo">412</span> LOG.debug(String.format("Wrote test data to file: %s", inputPath));<a name="line.412"></a>
+<span class="sourceLineNo">413</span><a name="line.413"></a>
+<span class="sourceLineNo">414</span> if (conf.getBoolean(FORCE_COMBINER_CONF, true)) {<a name="line.414"></a>
+<span class="sourceLineNo">415</span> LOG.debug("Forcing combiner.");<a name="line.415"></a>
+<span class="sourceLineNo">416</span> conf.setInt("mapreduce.map.combine.minspills", 1);<a name="line.416"></a>
+<span class="sourceLineNo">417</span> }<a name="line.417"></a>
+<span class="sourceLineNo">418</span><a name="line.418"></a>
+<span class="sourceLineNo">419</span> // Build args array.<a name="line.419"></a>
+<span class="sourceLineNo">420</span> String[] argsArray = new String[args.size() + 2];<a name="line.420"></a>
+<span class="sourceLineNo">421</span> Iterator it = args.entrySet().iterator();<a name="line.421"></a>
+<span class="sourceLineNo">422</span> int i = 0;<a name="line.422"></a>
+<span class="sourceLineNo">423</span> while (it.hasNext()) {<a name="line.423"></a>
+<span class="sourceLineNo">424</span> Map.Entry pair = (Map.Entry) it.next();<a name="line.424"></a>
+<span class="sourceLineNo">425</span> argsArray[i] = "-D" + pair.getKey() + "=" + pair.getValue();<a name="line.425"></a>
+<span class="sourceLineNo">426</span> i++;<a name="line.426"></a>
+<span class="sourceLineNo">427</span> }<a name="line.427"></a>
+<span class="sourceLineNo">428</span> argsArray[i] = table.getNameAsString();<a name="line.428"></a>
+<span class="sourceLineNo">429</span> argsArray[i + 1] = inputPath.toString();<a name="line.429"></a>
+<span class="sourceLineNo">430</span><a name="line.430"></a>
+<span class="sourceLineNo">431</span> // run the import<a name="line.431"></a>
+<span class="sourceLineNo">432</span> Tool tool = new ImportTsv();<a name="line.432"></a>
+<span class="sourceLineNo">433</span> LOG.debug("Running ImportTsv with arguments: " + argsArray);<a name="line.433"></a>
+<span class="sourceLineNo">434</span> assertEquals(0, ToolRunner.run(conf, tool, argsArray));<a name="line.434"></a>
+<span class="sourceLineNo">435</span><a name="line.435"></a>
+<span class="sourceLineNo">436</span> // Perform basic validation. If the input args did not include<a name="line.436"></a>
+<span class="sourceLineNo">437</span> // ImportTsv.BULK_OUTPUT_CONF_KEY then validate data in the table.<a name="line.437"></a>
+<span class="sourceLineNo">438</span> // Otherwise, validate presence of hfiles.<a name="line.438"></a>
+<span class="sourceLineNo">439</span> boolean isDryRun = args.containsKey(ImportTsv.DRY_RUN_CONF_KEY) &&<a name="line.439"></a>
+<span class="sourceLineNo">440</span> "true".equalsIgnoreCase(args.get(ImportTsv.DRY_RUN_CONF_KEY));<a name="line.440"></a>
+<span class="sourceLineNo">441</span> if (args.containsKey(ImportTsv.BULK_OUTPUT_CONF_KEY)) {<a name="line.441"></a>
+<span class="sourceLineNo">442</span> if (isDryRun) {<a name="line.442"></a>
+<span class="sourceLineNo">443</span> assertFalse(String.format("Dry run mode, %s should not have been created.",<a name="line.443"></a>
+<span class="sourceLineNo">444</span> ImportTsv.BULK_OUTPUT_CONF_KEY),<a name="line.444"></a>
+<span class="sourceLineNo">445</span> fs.exists(new Path(ImportTsv.BULK_OUTPUT_CONF_KEY)));<a name="line.445"></a>
+<span class="sourceLineNo">446</span> } else {<a name="line.446"></a>
+<span class="sourceLineNo">447</span> validateHFiles(fs, args.get(ImportTsv.BULK_OUTPUT_CONF_KEY), family,expectedKVCount);<a name="line.447"></a>
+<span class="sourceLineNo">448</span> }<a name="line.448"></a>
+<span class="sourceLineNo">449</span> } else {<a name="line.449"></a>
+<span class="sourceLineNo">450</span> validateTable(conf, table, family, valueMultiplier, isDryRun);<a name="line.450"></a>
+<span class="sourceLineNo">451</span> }<a name="line.451"></a>
<span class="sourceLineNo">452</span><a name="line.452"></a>
-<span class="sourceLineNo">453</span> LOG.debug("Validating table.");<a name="line.453"></a>
-<span class="sourceLineNo">454</span> Connection connection = ConnectionFactory.createConnection(conf);<a name="line.454"></a>
-<span class="sourceLineNo">455</span> Table table = connection.getTable(tableName);<a name="line.455"></a>
-<span class="sourceLineNo">456</span> boolean verified = false;<a name="line.456"></a>
-<span class="sourceLineNo">457</span> long pause = conf.getLong("hbase.client.pause", 5 * 1000);<a name="line.457"></a>
-<span class="sourceLineNo">458</span> int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);<a name="line.458"></a>
-<span class="sourceLineNo">459</span> for (int i = 0; i < numRetries; i++) {<a name="line.459"></a>
-<span class="sourceLineNo">460</span> try {<a name="line.460"></a>
-<span class="sourceLineNo">461</span> Scan scan = new Scan();<a name="line.461"></a>
-<span class="sourceLineNo">462</span> // Scan entire family.<a name="line.462"></a>
-<span class="sourceLineNo">463</span> scan.addFamily(Bytes.toBytes(family));<a name="line.463"></a>
-<span class="sourceLineNo">464</span> ResultScanner resScanner = table.getScanner(scan);<a name="line.464"></a>
-<span class="sourceLineNo">465</span> int numRows = 0;<a name="line.465"></a>
-<span class="sourceLineNo">466</span> for (Result res : resScanner) {<a name="line.466"></a>
-<span class="sourceLineNo">467</span> numRows++;<a name="line.467"></a>
-<span class="sourceLineNo">468</span> assertEquals(2, res.size());<a name="line.468"></a>
-<span class="sourceLineNo">469</span> List<Cell> kvs = res.listCells();<a name="line.469"></a>
-<span class="sourceLineNo">470</span> assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));<a name="line.470"></a>
-<span class="sourceLineNo">471</span> assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));<a name="line.471"></a>
-<span class="sourceLineNo">472</span> assertTrue(CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));<a name="line.472"></a>
-<span class="sourceLineNo">473</span> assertTrue(CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));<a name="line.473"></a>
-<span class="sourceLineNo">474</span> // Only one result set is expected, so let it loop.<a name="line.474"></a>
-<span class="sourceLineNo">475</span> }<a name="line.475"></a>
-<span class="sourceLineNo">476</span> if (isDryRun) {<a name="line.476"></a>
-<span class="sourceLineNo">477</span> assertEquals(0, numRows);<a name="line.477"></a>
-<span class="sourceLineNo">478</span> } else {<a name="line.478"></a>
-<span class="sourceLineNo">479</span> assertEquals(1, numRows);<a name="line.479"></a>
-<span class="sourceLineNo">480</span> }<a name="line.480"></a>
-<span class="sourceLineNo">481</span> verified = true;<a name="line.481"></a>
-<span class="sourceLineNo">482</span> break;<a name="line.482"></a>
-<span class="sourceLineNo">483</span> } catch (NullPointerException e) {<a name="line.483"></a>
-<span class="sourceLineNo">484</span> // If here, a cell was empty. Presume its because updates came in<a name="line.484"></a>
-<span class="sourceLineNo">485</span> // after the scanner had been opened. Wait a while and retry.<a name="line.485"></a>
-<span class="sourceLineNo">486</span> }<a name="line.486"></a>
-<span class="sourceLineNo">487</span> try {<a name="line.487"></a>
-<span class="sourceLineNo">488</span> Thread.sleep(pause);<a name="line.488"></a>
-<span class="sourceLineNo">489</span> } catch (InterruptedException e) {<a name="line.489"></a>
-<span class="sourceLineNo">490</span> // continue<a name="line.490"></a>
-<span class="sourceLineNo">491</span> }<a name="line.491"></a>
-<span class="sourceLineNo">492</span> }<a name="line.492"></a>
-<span class="sourceLineNo">493</span> table.close();<a name="line.493"></a>
-<span class="sourceLineNo">494</span> connection.close();<a name="line.494"></a>
-<span class="sourceLineNo">495</span> assertTrue(verified);<a name="line.495"></a>
-<span class="sourceLineNo">496</span> }<a name="line.496"></a>
-<span class="sourceLineNo">497</span><a name="line.497"></a>
-<span class="sourceLineNo">498</span> /**<a name="line.498"></a>
-<span class="sourceLineNo">499</span> * Confirm ImportTsv via HFiles on fs.<a name="line.499"></a>
-<span class="sourceLineNo">500</span> */<a name="line.500"></a>
-<span class="sourceLineNo">501</span> private static void validateHFiles(FileSystem fs, String outputPath, String family,<a name="line.501"></a>
-<span class="sourceLineNo">502</span> int expectedKVCount) throws IOException {<a name="line.502"></a>
-<span class="sourceLineNo">503</span> // validate number and content of output columns<a name="line.503"></a>
-<span class="sourceLineNo">504</span> LOG.debug("Validating HFiles.");<a name="line.504"></a>
-<span class="sourceLineNo">505</span> Set<String> configFamilies = new HashSet<String>();<a name="line.505"></a>
-<span class="sourceLineNo">506</span> configFamilies.add(family);<a name="line.506"></a>
-<span class="sourceLineNo">507</span> Set<String> foundFamilies = new HashSet<String>();<a name="line.507"></a>
-<span class="sourceLineNo">508</span> int actualKVCount = 0;<a name="line.508"></a>
-<span class="sourceLineNo">509</span> for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {<a name="line.509"></a>
-<span class="sourceLineNo">510</span> String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);<a name="line.510"></a>
-<span class="sourceLineNo">511</span> String cf = elements[elements.length - 1];<a name="line.511"></a>
-<span class="sourceLineNo">512</span> foundFamilies.add(cf);<a name="line.512"></a>
-<span class="sourceLineNo">513</span> assertTrue(<a name="line.513"></a>
-<span class="sourceLineNo">514</span> String.format(<a name="line.514"></a>
-<span class="sourceLineNo">515</span> "HFile output contains a column family (%s) not present in input families (%s)",<a name="line.515"></a>
-<span class="sourceLineNo">516</span> cf, configFamilies),<a name="line.516"></a>
-<span class="sourceLineNo">517</span> configFamilies.contains(cf));<a name="line.517"></a>
-<span class="sourceLineNo">518</span> for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {<a name="line.518"></a>
-<span class="sourceLineNo">519</span> assertTrue(<a name="line.519"></a>
-<span class="sourceLineNo">520</span> String.format("HFile %s appears to contain no data.", hfile.getPath()),<a name="line.520"></a>
-<span class="sourceLineNo">521</span> hfile.getLen() > 0);<a name="line.521"></a>
-<span class="sourceLineNo">522</span> // count the number of KVs from all the hfiles<a name="line.522"></a>
-<span class="sourceLineNo">523</span> if (expectedKVCount > -1) {<a name="line.523"></a>
-<span class="sourceLineNo">524</span> actualKVCount += getKVCountFromHfile(fs, hfile.getPath());<a name="line.524"></a>
-<span class="sourceLineNo">525</span> }<a name="line.525"></a>
-<span class="sourceLineNo">526</span> }<a name="line.526"></a>
-<span class="sourceLineNo">527</span> }<a name="line.527"></a>
-<span class="sourceLineNo">528</span> assertTrue(String.format("HFile output does not contain the input family '%s'.", family),<a name="line.528"></a>
-<span class="sourceLineNo">529</span> foundFamilies.contains(family));<a name="line.529"></a>
-<span class="sourceLineNo">530</span> if (expectedKVCount > -1) {<a name="line.530"></a>
-<span class="sourceLineNo">531</span> assertTrue(String.format(<a name="line.531"></a>
-<span class="sourceLineNo">532</span> "KV count in ouput hfile=<%d> doesn't match with expected KV count=<%d>", actualKVCount,<a name="line.532"></a>
-<span class="sourceLineNo">533</span> expectedKVCount), actualKVCount == expectedKVCount);<a name="line.533"></a>
-<span class="sourceLineNo">534</span> }<a name="line.534"></a>
-<span class="sourceLineNo">535</span> }<a name="line.535"></a>
-<span class="sourceLineNo">536</span><a name="line.536"></a>
-<span class="sourceLineNo">537</span> /**<a name="line.537"></a>
-<span class="sourceLineNo">538</span> * Method returns the total KVs in given hfile<a name="line.538"></a>
-<span class="sourceLineNo">539</span> * @param fs File System<a name="line.539"></a>
-<span class="sourceLineNo">540</span> * @param p HFile path<a name="line.540"></a>
-<span class="sourceLineNo">541</span> * @return KV count in the given hfile<a name="line.541"></a>
-<span class="sourceLineNo">542</span> * @throws IOException<a name="line.542"></a>
-<span class="sourceLineNo">543</span> */<a name="line.543"></a>
-<span class="sourceLineNo">544</span> private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {<a name="line.544"></a>
-<span class="sourceLineNo">545</span> Configuration conf = util.getConfiguration();<a name="line.545"></a>
-<span class="sourceLineNo">546</span> HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);<a name="line.546"></a>
-<span class="sourceLineNo">547</span> reader.loadFileInfo();<a name="line.547"></a>
-<span class="sourceLineNo">548</span> HFileScanner scanner = reader.getScanner(false, false);<a name="line.548"></a>
-<span class="sourceLineNo">549</span> scanner.seekTo();<a name="line.549"></a>
-<span class="sourceLineNo">550</span> int count = 0;<a name="line.550"></a>
-<span class="sourceLineNo">551</span> do {<a name="line.551"></a>
-<span class="sourceLineNo">552</span> count++;<a name="line.552"></a>
-<span class="sourceLineNo">553</span> } while (scanner.next());<a name="line.553"></a>
-<span class="sourceLineNo">554</span> reader.close();<a name="line.554"></a>
-<span class="sourceLineNo">555</span> return count;<a name="line.555"></a>
-<span class="sourceLineNo">556</span> }<a name="line.556"></a>
-<span class="sourceLineNo">557</span>}<a name="line.557"></a>
-<span class="sourceLineNo">558</span><a name="line.558"></a>
+<span class="sourceLineNo">453</span> if (conf.getBoolean(DELETE_AFTER_LOAD_CONF, true)) {<a name="line.453"></a>
+<span class="sourceLineNo">454</span> LOG.debug("Deleting test subdirectory");<a name="line.454"></a>
+<span class="sourceLineNo">455</span> util.cleanupDataTestDirOnTestFS(table.getNameAsString());<a name="line.455"></a>
+<span class="sourceLineNo">456</span> }<a name="line.456"></a>
+<span class="sourceLineNo">457</span> return tool;<a name="line.457"></a>
+<span class="sourceLineNo">458</span> }<a name="line.458"></a>
+<span class="sourceLineNo">459</span><a name="line.459"></a>
+<span class="sourceLineNo">460</span> /**<a name="line.460"></a>
+<span class="sourceLineNo">461</span> * Confirm ImportTsv via data in online table.<a name="line.461"></a>
+<span class="sourceLineNo">462</span> */<a name="line.462"></a>
+<span class="sourceLineNo">463</span> private static void validateTable(Configuration conf, TableName tableName,<a name="line.463"></a>
+<span class="sourceLineNo">464</span> String family, int valueMultiplier, boolean isDryRun) throws IOException {<a name="line.464"></a>
+<span class="sourceLineNo">465</span><a name="line.465"></a>
+<span class="sourceLineNo">466</span> LOG.debug("Validating table.");<a name="line.466"></a>
+<span class="sourceLineNo">467</span> Connection connection = ConnectionFactory.createConnection(conf);<a name="line.467"></a>
+<span class="sourceLineNo">468</span> Table table = connection.getTable(tableName);<a name="line.468"></a>
+<span class="sourceLineNo">469</span> boolean verified = false;<a name="line.469"></a>
+<span class="sourceLineNo">470</span> long pause = conf.getLong("hbase.client.pause", 5 * 1000);<a name="line.470"></a>
+<span class="sourceLineNo">471</span> int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);<a name="line.471"></a>
+<span class="sourceLineNo">472</span> for (int i = 0; i < numRetries; i++) {<a name="line.472"></a>
+<span class="sourceLineNo">473</span> try {<a name="line.473"></a>
+<span class="sourceLineNo">474</span> Scan scan = new Scan();<a name="line.474"></a>
+<span class="sourceLineNo">475</span> // Scan entire family.<a name="line.475"></a>
+<span class="sourceLineNo">476</span> scan.addFamily(Bytes.toBytes(family));<a name="line.476"></a>
+<span class="sourceLineNo">477</span> ResultScanner resScanner = table.getScanner(scan);<a name="line.477"></a>
+<span class="sourceLineNo">478</span> int numRows = 0;<a name="line.478"></a>
+<span class="sourceLineNo">479</span> for (Result res : resScanner) {<a name="line.479"></a>
+<span class="sourceLineNo">480</span> numRows++;<a name="line.480"></a>
+<span class="sourceLineNo">481</span> assertEquals(2, res.size());<a name="line.481"></a>
+<span class="sourceLineNo">482</span> List<Cell> kvs = res.listCells();<a name="line.482"></a>
+<span class="sourceLineNo">483</span> assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));<a name="line.483"></a>
+<span class="sourceLineNo">484</span> assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));<a name="line.484"></a>
+<span class="sourceLineNo">485</span> assertTrue(CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));<a name="line.485"></a>
+<span class="sourceLineNo">486</span> assertTrue(CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));<a name="line.486"></a>
+<span class="sourceLineNo">487</span> // Only one result set is expected, so let it loop.<a name="line.487"></a>
+<span class="sourceLineNo">488</span> }<a name="line.488"></a>
+<span class="sourceLineNo">489</span> if (isDryRun) {<a name="line.489"></a>
+<span class="sourceLineNo">490</span> assertEquals(0, numRows);<a name="line.490"></a>
+<span class="sourceLineNo">491</span> } else {<a name="line.491"></a>
+<span class="sourceLineNo">492</span> assertEquals(1, numRows);<a name="line.492"></a>
+<span class="sourceLineNo">493</span> }<a name="line.493"></a>
+<span class="sourceLineNo">494</span> verified = true;<a name="line.494"></a>
+<span class="sourceLineNo">495</span> break;<a name="line.495"></a>
+<span class="sourceLineNo">496</span> } catch (NullPointerException e) {<a name="line.496"></a>
+<span class="sourceLineNo">497</span> // If here, a cell was empty. Presume its because updates came in<a name="line.497"></a>
+<span class="sourceLineNo">498</span> // after the scanner had been opened. Wait a while and retry.<a name="line.498"></a>
+<span class="sourceLineNo">499</span> }<a name="line.499"></a>
+<span class="sourceLineNo">500</span> try {<a name="line.500"></a>
+<span class="sourceLineNo">501</span> Thread.sleep(pause);<a name="line.501"></a>
+<span class="sourceLineNo">502</span> } catch (InterruptedException e) {<a name="line.502"></a>
+<span class="sourceLineNo">503</span> // continue<a name="line.503"></a>
+<span class="sourceLineNo">504</span> }<a name="line.504"></a>
+<span class="sourceLineNo">505</span> }<a name="line.505"></a>
+<span class="sourceLineNo">506</span> table.close();<a name="line.506"></a>
+<span class="sourceLineNo">507</span> connection.close();<a name="line.507"></a>
+<span class="sourceLineNo">508</span> assertTrue(verified);<a name="line.508"></a>
+<span class="sourceLineNo">509</span> }<a name="line.509"></a>
+<span class="sourceLineNo">510</span><a name="line.510"></a>
+<span class="sourceLineNo">511</span> /**<a name="line.511"></a>
+<span class="sourceLineNo">512</span> * Confirm ImportTsv via HFiles on fs.<a name="line.512"></a>
+<span class="sourceLineNo">513</span> */<a name="line.513"></a>
+<span class="sourceLineNo">514</span> private static void validateHFiles(FileSystem fs, String outputPath, String family,<a name="line.514"></a>
+<span class="sourceLineNo">515</span> int expectedKVCount) throws IOException {<a name="line.515"></a>
+<span class="sourceLineNo">516</span> // validate number and content of output columns<a name="line.516"></a>
+<span class="sourceLineNo">517</span> LOG.debug("Validating HFiles.");<a name="line.517"></a>
+<span class="sourceLineNo">518</span> Set<String> configFamilies = new HashSet<String>();<a name="line.518"></a>
+<span class="sourceLineNo">519</span> configFamilies.add(family);<a name="line.519"></a>
+<span class="sourceLineNo">520</span> Set<String> foundFamilies = new HashSet<String>();<a name="line.520"></a>
+<span class="sourceLineNo">521</span> int actualKVCount = 0;<a name="line.521"></a>
+<span class="sourceLineNo">522</span> for (FileStatus cfStatus : fs.listStatus(new Path(outputPath), new OutputFilesFilter())) {<a name="line.522"></a>
+<span class="sourceLineNo">523</span> String[] elements = cfStatus.getPath().toString().split(Path.SEPARATOR);<a name="line.523"></a>
+<span class="sourceLineNo">524</span> String cf = elements[elements.length - 1];<a name="line.524"></a>
+<span class="sourceLineNo">525</span> foundFamilies.add(cf);<a name="line.525"></a>
+<span class="sourceLineNo">526</span> assertTrue(<a name="line.526"></a>
+<span class="sourceLineNo">527</span> String.format(<a name="line.527"></a>
+<span class="sourceLineNo">528</span> "HFile output contains a column family (%s) not present in input families (%s)",<a name="line.528"></a>
+<span class="sourceLineNo">529</span> cf, configFamilies),<a name="line.529"></a>
+<span class="sourceLineNo">530</span> configFamilies.contains(cf));<a name="line.530"></a>
+<span class="sourceLineNo">531</span> for (FileStatus hfile : fs.listStatus(cfStatus.getPath())) {<a name="line.531"></a>
+<span class="sourceLineNo">532</span> assertTrue(<a name="line.532"></a>
+<span class="sourceLineNo">533</span> String.format("HFile %s appears to contain no data.", hfile.getPath()),<a name="line.533"></a>
+<span class="sourceLineNo">534</span> hfile.getLen() > 0);<a name="line.534"></a>
+<span class="sourceLineNo">535</span> // count the number of KVs from all the hfiles<a name="line.535"></a>
+<span class="sourceLineNo">536</span> if (expectedKVCount > -1) {<a name="line.536"></a>
+<span class="sourceLineNo">537</span> actualKVCount += getKVCountFromHfile(fs, hfile.getPath());<a name="line.537"></a>
+<span class="sourceLineNo">538</span> }<a name="line.538"></a>
+<span class="sourceLineNo">539</span> }<a name="line.539"></a>
+<span class="sourceLineNo">540</span> }<a name="line.540"></a>
+<span class="sourceLineNo">541</span> assertTrue(String.format("HFile output does not contain the input family '%s'.", family),<a name="line.541"></a>
+<span class="sourceLineNo">542</span> foundFamilies.contains(family));<a name="line.542"></a>
+<span class="sourceLineNo">543</span> if (expectedKVCount > -1) {<a name="line.543"></a>
+<span class="sourceLineNo">544</span> assertTrue(String.format(<a name="line.544"></a>
+<span class="sourceLineNo">545</span> "KV count in ouput hfile=<%d> doesn't match with expected KV count=<%d>", actualKVCount,<a name="line.545"></a>
+<span class="sourceLineNo">546</span> expectedKVCount), actualKVCount == expectedKVCount);<a name="line.546"></a>
+<span class="sourceLineNo">547</span> }<a name="line.547"></a>
+<span class="sourceLineNo">548</span> }<a name="line.548"></a>
+<span class="sourceLineNo">549</span><a name="line.549"></a>
+<span class="sourceLineNo">550</span> /**<a name="line.550"></a>
+<span class="sourceLineNo">551</span> * Method returns the total KVs in given hfile<a name="line.551"></a>
+<span class="sourceLineNo">552</span> * @param fs File System<a name="line.552"></a>
+<span class="sourceLineNo">553</span> * @param p HFile path<a name="line.553"></a>
+<span class="sourceLineNo">554</span> * @return KV count in the given hfile<a name="line.554"></a>
+<span class="sourceLineNo">555</span> * @throws IOException<a name="line.555"></a>
+<span class="sourceLineNo">556</span> */<a name="line.556"></a>
+<span class="sourceLineNo">557</span> private static int getKVCountFromHfile(FileSystem fs, Path p) throws IOException {<a name="line.557"></a>
+<span class="sourceLineNo">558</span> Configuration conf = util.getConfiguration();<a name="line.558"></a>
+<span class="sourceLineNo">559</span> HFile.Reader reader = HFile.createReader(fs, p, new CacheConfig(conf), conf);<a name="line.559"></a>
+<span class="sourceLineNo">560</span> reader.loadFileInfo();<a name="line.560"></a>
+<span class="sourceLineNo">561</span> HFileScanner scanner = reader.getScanner(false, false);<a name="line.561"></a>
+<span class="sourceLineNo">562</span> scanner.seekTo();<a name="line.562"></a>
+<span class="sourceLineNo">563</span> int count = 0;<a name="line.563"></a>
+<span class="sourceLineNo">564</span> do {<a name="line.564"></a>
+<span class="sourceLineNo">565</span> count++;<a name="line.565"></a>
+<span class="sourceLineNo">566</span> } while (scanner.next());<a name="line.566"></a>
+<span class="sourceLineNo">567</span> reader.close();<a name="line.567"></a>
+<span class="sourceLineNo">568</span> return count;<a name="line.568"></a>
+<span class="sourceLineNo">569</span> }<a name="line.569"></a>
+<span class="sourceLineNo">570</span>}<a name="line.570"></a>
+<span class="sourceLineNo">571</span><a name="line.571"></a>